diff -Nru cython-0.20.1+1~201611251650-6686/2to3-fixers.txt cython-0.20.1+1~202203241016-9537/2to3-fixers.txt --- cython-0.20.1+1~201611251650-6686/2to3-fixers.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/2to3-fixers.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -lib2to3.fixes.fix_unicode diff -Nru cython-0.20.1+1~201611251650-6686/appveyor/install.ps1 cython-0.20.1+1~202203241016-9537/appveyor/install.ps1 --- cython-0.20.1+1~201611251650-6686/appveyor/install.ps1 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/appveyor/install.ps1 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ # Sample script to install Python and pip under Windows # Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +# License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ $PYTHON_BASE_URL = "https://www.python.org/ftp/python/" $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" @@ -48,12 +48,14 @@ $installer_exe = ($py_major + $py_minor) -as [int] -ge 35 if ($installer_exe) { $arch_suffix = @{"32"="";"64"="-amd64"}[$architecture] - $filename = "python-" + $python_version + $arch_suffix + ".exe" + $dl_filename = "python-" + $python_version + $arch_suffix + ".exe" + $filename = "python-" + $py_major + "." + $py_minor + $arch_suffix + ".exe" } else { $arch_suffix = @{"32"="";"64"=".amd64"}[$architecture] - $filename = "python-" + $python_version + $arch_suffix + ".msi" + $dl_filename = "python-" + $python_version + $arch_suffix + ".msi" + $filename = "python-" + $py_major + "." + $py_minor + $arch_suffix + ".msi" } - $url = $PYTHON_BASE_URL + $python_version + "/" + $filename + $url = $PYTHON_BASE_URL + $python_version + "/" + $dl_filename $filepath = Download $url $filename $DOWNLOADS Write-Host "Installing" $filename "to" $python_home if ($installer_exe) { @@ -99,7 +101,8 @@ } function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + $full_version = $env:PYTHON_VERSION + ".0" + InstallPython $full_version $env:PYTHON_ARCH $env:PYTHON InstallPip $env:PYTHON InstallPipPackage $env:PYTHON setuptools InstallPipPackage $env:PYTHON wheel diff -Nru cython-0.20.1+1~201611251650-6686/appveyor.yml cython-0.20.1+1~202203241016-9537/appveyor.yml --- cython-0.20.1+1~201611251650-6686/appveyor.yml 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/appveyor.yml 2022-03-24 10:16:46.000000000 +0000 @@ -4,57 +4,107 @@ global: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 + # /E:ON and /V:ON options are not enabled in the batch script interpreter + # See: https://stackoverflow.com/questions/11267463/compiling-python-modules-on-windows-x64/13751649#13751649 WITH_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" + BACKEND: c + PARALLEL: "-j4" + EXTRA_CFLAGS: "" matrix: - - PYTHON: "C:\\Python26" - PYTHON_VERSION: "2.6.6" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7.9" + PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.5" + - PYTHON: "C:\\Python39" + PYTHON_VERSION: "3.9" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.3" + - PYTHON: "C:\\Python39-x64" + PYTHON_VERSION: "3.9" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python38" + PYTHON_VERSION: "3.8" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python35" - PYTHON_VERSION: "3.5.0" + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8" + PYTHON_ARCH: "64" + EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" + + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8" + PYTHON_ARCH: "64" + BACKEND: c,cpp + + - PYTHON: "C:\\Python37" + PYTHON_VERSION: "3.7" PYTHON_ARCH: "32" + BACKEND: c,cpp - - PYTHON: "C:\\Python26-x64" - PYTHON_VERSION: "2.6.6" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.9" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" + EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" - - PYTHON: "C:\\Python33-x64" - PYTHON_VERSION: "3.3.5" + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" + BACKEND: cpp - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.3" + - PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python35" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "32" + - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.0" + PYTHON_VERSION: "3.5" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python34" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "32" + PARALLEL: "" + + - PYTHON: "C:\\Python34-x64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + PARALLEL: "" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" + BACKEND: cpp + PYTHONIOENCODING: "utf-8" + PARALLEL: "" clone_depth: 5 branches: only: - master - - 0.23.x + - release + - 0.29.x init: - "ECHO Python %PYTHON_VERSION% (%PYTHON_ARCH%bit) from %PYTHON%" @@ -67,11 +117,16 @@ build: off build_script: + - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext %PARALLEL%" + - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext --inplace" - "%WITH_ENV% %PYTHON%\\python.exe setup.py bdist_wheel" test: off test_script: - - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --no-cpp" + - "%PYTHON%\\Scripts\\pip.exe install -r test-requirements.txt" + - "%PYTHON%\\Scripts\\pip.exe install win_unicode_console" + - "set CFLAGS=/Od /W3 %EXTRA_CFLAGS%" + - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --backend=%BACKEND% --no-code-style -j5" artifacts: - path: dist\* diff -Nru cython-0.20.1+1~201611251650-6686/bin/cython_freeze cython-0.20.1+1~202203241016-9537/bin/cython_freeze --- cython-0.20.1+1~201611251650-6686/bin/cython_freeze 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/bin/cython_freeze 2022-03-24 10:16:46.000000000 +0000 @@ -3,7 +3,7 @@ Create a C file for embedding one or more Cython source files. Requires Cython 0.11.2 (or perhaps newer). -See Demos/freeze/README.txt for more details. +See Demos/freeze/README.rst for more details. """ from __future__ import print_function diff -Nru cython-0.20.1+1~201611251650-6686/bin/cython-generate-lexicon.py cython-0.20.1+1~202203241016-9537/bin/cython-generate-lexicon.py --- cython-0.20.1+1~201611251650-6686/bin/cython-generate-lexicon.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/bin/cython-generate-lexicon.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# +# Updates Cython's Lexicon.py with the unicode characters that are accepted as +# identifiers. Should be run with the most recent version of Python possible +# to ensure that Lexicon is as complete as possible. +# +# Python3 only (it relies on str.isidentifier which is a Python 3 addition) +# +# Run with either +# --overwrite to update the existing Lexicon.py file +# --here to create a copy of Lexicon.py in the current directory + +import functools +import re +import os +import sys + +# Make sure we import the right Cython +cythonpath, _ = os.path.split(os.path.realpath(__file__)) # bin directory +cythonpath, _ = os.path.split(cythonpath) +if os.path.exists(os.path.join(cythonpath, "Cython")): + sys.path.insert(0, cythonpath) + print("Found (and using) local cython directory") +# else we aren't in a development directory + +from Cython.Compiler import Lexicon + + +def main(): + arg = '--overwrite' + if len(sys.argv) == 2: + arg = sys.argv[1] + if len(sys.argv) > 2 or arg not in ['--overwrite','--here']: + print("""Call the script with either: + --overwrite to update the existing Lexicon.py file (default) + --here to create an version of Lexicon.py in the current directory +""") + return + + generated_code = ( + f"# generated with:\n" + f"# {sys.implementation.name} {sys.version.splitlines()[0].strip()}\n" + "\n" + f"{generate_character_sets()}\n" + ) + + print("Reading file", Lexicon.__file__) + with open(Lexicon.__file__, 'r') as f: + parts = re.split(r"(# (?:BEGIN|END) GENERATED CODE\n?)", f.read()) + + if len(parts) not in (4,5) or ' GENERATED CODE' not in parts[1] or ' GENERATED CODE' not in parts[3]: + print("Warning: generated code section not found - code not inserted") + return + + parts[2] = generated_code + output = "".join(parts) + + if arg == "--here": + outfile = "Lexicon.py" + else: + assert arg == "--overwrite" + outfile = Lexicon.__file__ + + print("Writing to file", outfile) + with open(outfile, 'w') as f: + f.write(output) + + +# The easiest way to generate an appropriate character set is just to use the str.isidentifier method +# An alternative approach for getting character sets is at https://stackoverflow.com/a/49332214/4657412 +@functools.lru_cache() +def get_start_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier(chr(i)) ] + + +def get_continue_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier('a'+chr(i)) ] + + +def get_continue_not_start_as_number(): + start = get_start_characters_as_number() + cont = get_continue_characters_as_number() + assert set(start) <= set(cont), \ + "We assume that all identifier start characters are also continuation characters." + return sorted(set(cont).difference(start)) + + +def to_ranges(char_num_list): + # Convert the large lists of character digits to + # list of characters + # a list pairs of characters representing closed ranges + char_num_list = sorted(char_num_list) + first_good_val = char_num_list[0] + + single_chars = [] + ranges = [] + for n in range(1, len(char_num_list)): + if char_num_list[n]-1 != char_num_list[n-1]: + # discontinuous + if first_good_val == char_num_list[n-1]: + single_chars.append(chr(char_num_list[n-1])) + else: + ranges.append(chr(first_good_val) + chr(char_num_list[n-1])) + first_good_val = char_num_list[n] + + return ''.join(single_chars), ''.join(ranges) + + +def make_split_strings(chars, splitby=60, indent=" "): + lines = [f'u"{chars[i:i+splitby]}"' for i in range(0, len(chars), splitby)] + return indent + f"\n{indent}".join(lines) + + +def generate_character_sets(): + declarations = [] + for char_type, char_generator in [ + ("unicode_start_ch", get_start_characters_as_number), + ("unicode_continuation_ch", get_continue_not_start_as_number), + ]: + for set_type, chars in zip(("any", "range"), to_ranges(char_generator())): + declarations.append( + f"{char_type}_{set_type} = (\n" + f"{make_split_strings(chars)}\n" + f")\n" + ) + + return "".join(declarations) + + +if __name__ == "__main__": + main() diff -Nru cython-0.20.1+1~201611251650-6686/bin/cythonrun cython-0.20.1+1~202203241016-9537/bin/cythonrun --- cython-0.20.1+1~201611251650-6686/bin/cythonrun 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/bin/cythonrun 2022-03-24 10:16:46.000000000 +0000 @@ -9,7 +9,7 @@ python cythonrun somefile.py [ARGS] """ -from Cython.Build.BuildExecutable import build, build_and_run +from Cython.Build.BuildExecutable import build_and_run if __name__ == '__main__': import sys diff -Nru cython-0.20.1+1~201611251650-6686/CHANGES.rst cython-0.20.1+1~202203241016-9537/CHANGES.rst --- cython-0.20.1+1~201611251650-6686/CHANGES.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/CHANGES.rst 2022-03-24 10:16:46.000000000 +0000 @@ -2,7 +2,2625 @@ Cython Changelog ================ -0.25 (2016-??-??) +3.0.0 alpha 11 (2022-0?-??) +=========================== + +Features added +-------------- + +* A new decorator ``@cython.dataclasses.dataclass`` was implemented that provides + compile time dataclass generation capabilities to ``cdef`` classes (extension types). + Patch by David Woods. (Github issue :issue:`2903`) + +* Named expressions (PEP 572) aka. assignment expressions (aka. the walrus operator + ``:=``) were implemented. + Patch by David Woods. (Github issue :issue:`2636`) + +* The ``cythonize`` command has a new option ``-M`` to generate ``.dep`` dependency + files for the compilation unit. This can be used by external build tools to track + these dependencies. Already available in Cython :ref:`0.29.27`. + Patch by Evgeni Burovski. (Github issue :issue:`1214`) + +* ``cythonize()`` and the corresponding CLI command now regenerate the output files + also when they already exist but were generated by a different Cython version. + +* The environment variable ``CYTHON_FORCE_REGEN=1`` can be used to force ``cythonize`` + to regenerate the output files regardless of modification times and changes. + +* The generated C code now compiles in CPython 3.11a4. + (Github issue :issue:`4500`) + +Bugs fixed +---------- + +* Includes all bug-fixes from the :ref:`0.29.27` release. + +Other changes +------------- + +* When using type annotations, ``func(x: list)`` or ``func(x: ExtType)`` (and other + Python builtin or extension types) now disallow ``None`` as input argument to ``x``. + This is consistent with the normal typing semantics in Python, and was a common gotcha + for users who did not expect ``None`` to be allowed as input. To allow ``None``, use + ``typing.Optional`` as in ``func(x: Optional[list])``. ``None`` is also automatically + allowed when it is used as default argument, i.e. ``func(x: list = None)``. + Note that, for backwards compatibility reasons, this does not apply when using Cython's + C notation, as in ``func(list x)``. Here, ``None`` is still allowed, as always. + (Github issues :issue:`3883`, :issue:`2696`) + + +3.0.0 alpha 10 (2022-01-06) +=========================== + +Features added +-------------- + +* ``Cython.Distutils.build_ext`` now uses ``cythonize()`` internally (previously + known as ``new_build_ext``), while still supporting the options that were + available in the old implementation (``old_build_ext``). + Patch by Matus Valo. (Github issue :issue:`3541`) + +* ``pyximport`` now uses ``cythonize()`` internally. + Patch by Matus Valo. (Github issue :issue:`2304`) + +* ``__del__(self)`` on extension types now maps to ``tp_finalize`` in Python 3. + Original patch by ax487. (Github issue :issue:`3612`) + +* Conversion from Python dict to C++ map now supports arbitrary Python mappings, + not just dicts. + +* Direct assignments to C++ references are now allowed. + Patch by David Woods. (Github issue :issue:`1863`) + +* An initial set of adaptations for GraalVM Python was implemented. Note that + this does not imply any general support for this target or that your code + will work at all in this environment. But testing should be possible now. + Patch by David Woods. (Github issue :issue:`4328`) + +* ``PyMem_[Raw]Calloc()`` was added to the ``cpython.mem`` declarations. + Note that the ``Raw`` versions are no longer #defined by Cython. The previous + macros were not considered safe. + Patch by William Schwartz and David Woods. (Github issue :issue:`3047`) + +Bugs fixed +---------- + +* Circular imports of compiled modules could fail needlessly even when the import + could already be resolved from ``sys.modules``. + Patch by Syam Gadde. (Github issue :issue:`4390`) + +* The GIL can now safely be released inside of ``nogil`` functions (which may actually + be called with the GIL held at runtime). + Patch by David Woods. (Github issue :issue:`4137`) + +* Type errors when passing memory view arguments could leak buffer references. + Patch by David Woods. (Github issue :issue:`4296`) + +* Cython did not type the ``self`` argument in special binary methods. + Patch by David Woods. (Github issue :issue:`4434`) + +* An incompatibility with recent coverage.py versions was resolved. + Patch by David Woods. (Github issue :issue:`4440`) + +* Fused typed default arguments generated incorrect code. + Patch by David Woods. (Github issue :issue:`4413`) + +* ``prange`` loops generated incorrect code when ``cpp_locals`` is enabled. + Patch by David Woods. (Github issue :issue:`4354`) + +* A C-level compatibility issue with recent NumPy versions was resolved. + Patch by David Woods. (Github issue :issue:`4396`) + +* Decorators on inner functions were not evaluated in the right scope. + Patch by David Woods. (Github issue :issue:`4367`) + +* Very early errors during module initialisation could lead to crashes. + Patch by David Woods. (Github issue :issue:`4377`) + +* Fused functions were binding unnecessarily, which prevented them from being pickled. + Patch by David Woods. (Github issue :issue:`4370`) + +* Some constant tuples containing strings were not deduplicated. + Patch by David Woods. (Github issue :issue:`4353`) + +* Unsupported decorators on cdef functions were not rejected in recent releases. + Patch by David Woods. (Github issue :issue:`4322`) + +* The excess arguments in a for-in-range loop with more than 3 arguments to `range()` + were silently ignored. + Original patch by Max Bachmann. (Github issue :issue:`4550`) + +* Python object types were not allowed as ``->`` return type annotations. + Patch by Matus Valo. (Github issue :issue:`4433`) + +* Default values for memory views arguments were not properly supported. + Patch by Corentin Cadiou. (Github issue :issue:`4313`) + +* Templating C++ classes with memory view types lead to buggy code and is now rejected. + Patch by David Woods. (Github issue :issue:`3085`) + +* Several C++ library declarations were added and fixed. + Patches by Dobatymo, account-login, Jonathan Helgert, Evgeny Yakimov, GalaxySnail, Max Bachmann. + (Github issues :issue:`4408`, :issue:`4419`, :issue:`4410`, :issue:`4395`, + :issue:`4423`, :issue:`4448`, :issue:`4462`, :issue:`3293`, :issue:`4522`, + :issue:`2171`, :issue:`4531`) + +* Some compiler problems and warnings were resolved. + Patches by David Woods, 0dminnimda, Nicolas Pauss and others. + (Github issues :issue:`4317`, :issue:`4324`, :issue:`4361`, :issue:`4357`) + +* The ``self`` argument of static methods in .pxd files was incorrectly typed. + Patch by David Woods. (Github issue :issue:`3174`) + +* A name collision when including multiple generated API header files was resolved. + Patch by David Woods. (Github issue :issue:`4308`) + +* An endless loop in ``cython-mode.el`` was resolved. + Patch by Johannes Mueller. (Github issue :issue:`3218`) + +* ``_Py_TPFLAGS_HAVE_VECTORCALL`` was always set on extension types when using the limited API. + Patch by David Woods. (Github issue :issue:`4453`) + +* Some compatibility issues with PyPy were resolved. + Patches by Max Bachmann, Matti Picus. + (Github issues :issue:`4454`, :issue:`4477`, :issue:`4478`, :issus:`4509`, :issue:`4517`) + +* A compiler crash when running Cython thread-parallel from distutils was resolved. + (Github issue :issue:`4503`) + +* Includes all bug-fixes from the :ref:`0.29.26` release. + +Other changes +------------- + +* A warning was added when ``__defaults__`` or ``__kwdefaults__`` of Cython compiled + functions were re-assigned, since this does not current have an effect. + Patch by David Woods. (Github issue :issue:`2650`) + + +3.0.0 alpha 9 (2021-07-21) +========================== + +Features added +-------------- + +* Declarations for ``libcpp.algorithms``, ``libcpp.set`` and ``libcpp.unordered_set`` + were extended. + Patch by David Woods. (Github issues :issue:`4271`, :issue:`4273`) + +* ``cygdb`` has a new option ``--skip-interpreter`` that allows using a different + Python runtime than the one used to generate the debugging information. + Patch by Alessandro Molina. (Github issue :issue:`4186`) + +Bugs fixed +---------- + +* Several issues with the new ``cpp_locals`` directive were resolved and + its test coverage improved. + Patch by David Woods. (Github issues :issue:`4266`, :issue:`4265`) + +* Generated utility code for C++ conversions no longer depends on several user + definable directives that may make it behave incorrectly. + Patch by David Woods. (Github issue :issue:`4206`) + +* A reference counting bug in the new ``@cython.total_ordering`` decorator was fixed. + +* Includes all bug-fixes from the :ref:`0.29.24` release. + +Other changes +------------- + +* Parts of the documentation were (and are being) rewritten to show the + Cython language syntax next to the equivalent Python syntax. + Patches by 0dminnimda and Matus Valo. (Github issue :issue:`4187`) + + +3.0.0 alpha 8 (2021-07-02) +========================== + +Features added +-------------- + +* A ``@cython.total_ordering`` decorator has been added to automatically + implement all comparison operators, similar to ``functools.total_ordering``. + Patch by Spencer Brown. (Github issue :issue:`2090`) + +* A new directive ``cpp_locals`` was added that allows local C++ variables to + be lazily initialised (without default constructor), thus making them behave + more like Python variables. + Patch by David Woods. (Github issue :issue:`4160`) + +* C++17 execution policies are supported in ``libcpp.algorithm``. + Patch by Ashwin Srinath. (Github issue :issue:`3790`) + +* New C feature flags: ``CYTHON_USE_MODULE_STATE``, ``CYTHON_USE_TYPE_SPECS`` + Both are currently considered experimental. + (Github issue :issue:`3611`) + +* ``[...] * N`` is optimised for C integer multipliers ``N``. + (Github issue :issue:`3922`) + +Bugs fixed +---------- + +* The dispatch code for binary operators to special methods could run into infinite recursion. + Patch by David Woods. (Github issue :issue:`4172`) + +* Code optimisations were not applied to methods of Cython implemented C++ classes. + Patch by David Woods. (Github issue :issue:`4212`) + +* The special ``cython`` module was not always detected in PEP-484 type annotations. + Patch by David Woods. (Github issue :issue:`4243`) + +* Conversion from Python dicts to ``std::map`` was broken. + Patch by David Woods and Mikkel Skofelt. (Github issues :issue:`4231`, :issue:`4228`) + +* The exception handling annotation ``except +*`` was broken. + Patch by David Woods. (Github issues :issue:`3065`, :issue:`3066`) + +* Attribute annotations in Python classes are now ignored, because they are + just Python objects in a dict (as opposed to the fields of extension types). + Patch by David Woods. (Github issues :issue:`4196`, :issue:`4198`) + +* An unnecessary slow-down at import time was removed from ``Cython.Distutils``. + Original patch by Anthony Sottile. (Github issue :issue:`4224`) + +* Python modules were not automatically recompiled when only their ``.pxd`` file changed. + Patch by Golden Rockefeller. (Github issue :issue:`1428`) + +* The signature of ``PyFloat_FromString()`` in ``cpython.float`` was changed + to match the signature in Py3. It still has an automatic fallback for Py2. + (Github issue :issue:`3909`) + +* A compile error on MSVC was resolved. + Patch by David Woods. (Github issue :issue:`4202`) + +* A C compiler warning in PyPy3 regarding ``PyEval_EvalCode()`` was resolved. + +* Directives starting with ``optimization.*`` in pure Python mode were incorrectly named. + It should have been ``optimize.*``. + Patch by David Woods. (Github issue :issue:`4258`) + +Other changes +------------- + +* Variables can no longer be declared with ``cpdef``. + Patch by David Woods. (Github issue :issue:`887`) + +* Support for the now unsupported Pyston V1 was removed in favour of Pyston V2. + Patch by Marius Wachtler. (Github issue :issue:`4211`) + +* The ``Cython.Build.BuildExecutable`` tool no longer executes the program automatically. + Use ``cythonrun`` for that. + + +3.0.0 alpha 7 (2021-05-24) +========================== + +Features added +-------------- + +* A ``cimport`` is now supported in pure Python code by prefixing the + imported module name with ``cython.cimports.``, e.g. + ``from cython.cimports.libc.math import sin``. + (GIthub issue :issue:`4190`) + +* ``__class_getitem__`` (`PEP-560`_) is supported for cdef classes. + Patch by Kmol Yuan. (Github issue :issue:`3764`) + +* ``__mro_entries__`` (`PEP-560`_) is supported for Python classes. + Patch by David Woods. (Github issue :issue:`3537`) + +* ``cython.array`` supports simple, non-strided views. + (Github issue :issue:`3775`) + +* Self-documenting f-strings (``=``) were implemented. + Patch by davfsa. (Github issue :issue:`3796`) + +* The destructor is now called for fields in C++ structs. + Patch by David Woods. (Github issue :issue:`3226`) + +* ``std::move()`` is now also called for temps during ``yield``. + Patch by Yu Feng. (Github issue :issue:`4154`) + +* ``asyncio.iscoroutinefunction()`` now recognises coroutine functions + also when compiled by Cython. + Patch by Pedro Marques da Luz. (Github issue :issue:`2273`) + +* C compiler warnings and errors are now shown in Jupyter notebooks. + Patch by Egor Dranischnikow. (Github issue :issue:`3751`) + +* ``float(…)`` is optimised for string arguments (str/bytes/bytearray). + +* Converting C++ containers to Python lists uses less memory allocations. + Patch by Max Bachmann. (Github issue :issue:`4081`) + +* Docstrings of ``cpdef`` enums are now copied to the enum class. + Patch by matham. (Github issue :issue:`3805`) + +* The type ``cython.Py_hash_t`` is available in Python mode. + +* C-API declarations for ``cpython.fileobject`` were added. + Patch by Zackery Spytz. (Github issue :issue:`3906`) + +* C-API declarations for context variables in Python 3.7 were added. + Original patch by Zolisa Bleki. (Github issue :issue:`2281`) + +* More C-API declarations for ``cpython.datetime`` were added. + Patch by Bluenix2. (Github issue :issue:`4128`) + +* A new module ``cpython.time`` was added with some low-level alternatives to + Python's ``time`` module. + Patch by Brock Mendel. (Github issue :issue:`3767`) + +* The value ``PyBUF_MAX_NDIM`` was added to the ``cpython.buffer`` module. + Patch by John Kirkham. (Github issue :issue:`3811`) + +* "Declaration after use" is now an error for variables. + Patch by David Woods. (Github issue :issue:`3976`) + +* More declarations for C++ string methods were added. + +* Cython now detects when existing output files were not previously generated + by itself and refuses to overwrite them. It is a common mistake to name + the module file of a wrapper after the library (source file) that it wraps, + which can lead to surprising errors when the file gets overwritten. + +Bugs fixed +---------- + +* Annotations were not exposed on annotated (data-)classes. + Patch by matsjoyce. (Github issue :issue:`4151`) + +* Inline functions and other code in ``.pxd`` files could accidentally + inherit the compiler directives of the ``.pyx`` file that imported them. + Patch by David Woods. (Github issue :issue:`1071`) + +* Some issues were resolved that could lead to duplicated C names. + Patch by David Woods. (Github issue :issue:`3716`, :issue:`3741`, :issue:`3734`) + +* Modules with unicode names failed to build on Windows. + Patch by David Woods. (Github issue :issue:`4125`) + +* ``ndarray.shape`` failed to compile with Pythran and recent NumPy. + Patch by Serge Guelton. (Github issue :issue:`3762`) + +* Casting to ctuples is now allowed. + Patch by David Woods. (Github issue :issue:`3808`) + +* Structs could not be instantiated with positional arguments in + pure Python mode. + +* Literal list assignments to pointer variables declared in PEP-526 + notation failed to compile. + +* Nested C++ types were not usable through ctypedefs. + Patch by Vadim Pushtaev. (Github issue :issue:`4039`) + +* Overloaded C++ static methods were lost. + Patch by Ashwin Srinath. (Github :issue:`1851`) + +* Cython compiled functions always provided a ``__self__`` attribute, + regardless of being used as a method or not. + Patch by David Woods. (Github issue :issue:`4036`) + +* Calls to ``.__class__()`` of a known extension type failed. + Patch by David Woods. (Github issue :issue:`3954`) + +* Generator expressions in pxd-overridden ``cdef`` functions could + fail to compile. + Patch by Matus Valo. (Github issue :issue:`3477`) + +* A reference leak on import failures was resolved. + Patch by Max Bachmann. (Github issue :issue:`4056`) + +* A C compiler warning about unused code was resolved. + (Github issue :issue:`3763`) + +* A C compiler warning about enum value casting was resolved in GCC. + (Github issue :issue:`2749`) + +* Some C compiler warninge were resolved. + Patches by Max Bachmann. (Github issue :issue:`4053`, :issue:`4059`, :issue:`4054`, :issue:`4148`, :issue:`4162`) + +* A compile failure for C++ enums in Py3.4 / MSVC was resolved. + Patch by Ashwin Srinath. (Github issue :issue:`3782`) + +* Some C++ STL methods did not propagate exceptions. + Patch by Max Bachmann. (Github issue :issue:`4079`) + +* An unsupported C-API call in PyPy was fixed. + Patch by Max Bachmann. (Github issue :issue:`4055`) + +* The Cython ``CodeWriter`` mishandled no-argument ``return`` statements. + Patch by Tao He. (Github issue :issue:`3795`) + +* ``complex`` wasn't supported in PEP-484 type annotations. + Patch by David Woods. (Github issue :issue:`3949`) + +* Default arguments of methods were not exposed for introspection. + Patch by Vladimir Matveev. (Github issue :issue:`4061`) + +* Extension types inheriting from Python classes could not safely + be exposed in ``.pxd`` files. + (Github issue :issue:`4106`) + +* The profiling/tracing code was adapted to work with Python 3.10b1. + +* The internal CPython macro ``Py_ISSPACE()`` is no longer used. + Original patch by Andrew Jones. (Github issue :issue:`4111`) + +* Includes all bug-fixes from the :ref:`0.29.23` release. + + +3.0.0 alpha 6 (2020-07-31) +========================== + +Features added +-------------- + +* Special methods for binary operators now follow Python semantics. + Rather than e.g. a single ``__add__`` method for cdef classes, where + "self" can be either the first or second argument, one can now define + both ``__add__`` and ``__radd__`` as for standard Python classes. + This behavior can be disabled with the ``c_api_binop_methods`` directive + to return to the previous semantics in Cython code (available from Cython + 0.29.20), or the reversed method (``__radd__``) can be implemented in + addition to an existing two-sided operator method (``__add__``) to get a + backwards compatible implementation. + (Github issue :issue:`2056`) + +* No/single argument functions now accept keyword arguments by default in order + to comply with Python semantics. The marginally faster calling conventions + ``METH_NOARGS`` and ``METH_O`` that reject keyword arguments are still available + with the directive ``@cython.always_allow_keywords(False)``. + (Github issue :issue:`3090`) + +* For-in-loop iteration over ``bytearray`` and memory views is optimised. + Patch by David Woods. (Github issue :issue:`2227`) + +* Type inference now works for memory views and slices. + Patch by David Woods. (Github issue :issue:`2227`) + +* The ``@returns()`` decorator propagates exceptions by default for suitable C + return types when no ``@exceptval()`` is defined. + (Github issues :issue:`3625`, :issue:`3664`) + +* A low-level inline function ``total_seconds(timedelta)`` was added to + ``cpython.datetime`` to bypass the Python method call. Note that this function + is not guaranteed to give exactly the same results for very large time intervals. + Patch by Brock Mendel. (Github issue :issue:`3616`) + +* Type inference now understands that ``a, *b = x`` assigns a list to ``b``. + +* Limited API support was improved. + Patches by Matthias Braun. (Github issues :issue:`3693`, :issue:`3707`) + +* The Cython ``CodeWriter`` can now handle more syntax constructs. + Patch by Tao He. (Github issue :issue:`3514`) + +Bugs fixed +---------- + +* The construct ``for x in cpp_function_call()`` failed to compile. + Patch by David Woods. (Github issue :issue:`3663`) + +* C++ references failed to compile when used as Python object indexes. + Patch by David Woods. (Github issue :issue:`3754`) + +* The C++ ``typeid()`` function was allowed in C mode. + Patch by Celelibi. (Github issue :issue:`3637`) + +* ``repr()`` was assumed to return ``str`` instead of ``unicode`` with ``language_level=3``. + (Github issue :issue:`3736`) + +* Includes all bug-fixes from the :ref:`0.29.21` release. + +Other changes +------------- + +* The ``numpy`` declarations were updated. + Patch by Brock Mendel. (Github issue :issue:`3630`) + +* The names of Cython's internal types (functions, generator, coroutine, etc.) + are now qualified with the module name of the internal Cython module that is + used for sharing them across Cython implemented modules, for example + ``_cython_3_0a5.coroutine``. This was done to avoid making them look like + homeless builtins, to help with debugging, and in order to avoid a CPython + warning according to https://bugs.python.org/issue20204 + +3.0.0 alpha 5 (2020-05-19) +========================== + +Features added +-------------- + +* ``.pxd`` files can now be :ref:`versioned ` by adding an + extension like "``.cython-30.pxd``" to prevent older Cython versions (than + 3.0 in this case) from picking them up. (Github issue :issue:`3577`) + +* Several macros/functions declared in the NumPy API are now usable without + holding the GIL. + +* `libc.math` was extended to include all C99 function declarations. + Patch by Dean Scarff. (Github issue :issue:`3570`) + +Bugs fixed +---------- + +* Several issues with arithmetic overflow handling were resolved, including + undefined behaviour in C. + Patch by Sam Sneddon. (Github issue :issue:`3588`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could fail to acquire the GIL in some cases on function exit. + (Github issue :issue:`3590` etc.) + +* A reference leak when processing keyword arguments in Py2 was resolved, + that appeared in 3.0a1. + (Github issue :issue:`3578`) + +* The outdated getbuffer/releasebuffer implementations in the NumPy + declarations were removed so that buffers declared as ``ndarray`` + now use the normal implementation in NumPy. + +* Includes all bug-fixes from the :ref:`0.29.18` release. + + +3.0.0 alpha 4 (2020-05-05) +========================== + +Features added +-------------- + +* The ``print`` statement (not the ``print()`` function) is allowed in + ``nogil`` code without an explicit ``with gil`` section. + +* The ``assert`` statement is allowed in ``nogil`` sections. Here, the GIL is + only acquired if the ``AssertionError`` is really raised, which means that the + evaluation of the asserted condition only allows C expressions. + +* Cython generates C compiler branch hints for unlikely user defined if-clauses + in more cases, when they end up raising exceptions unconditionally. This now + includes exceptions being raised in ``nogil``/``with gil`` sections. + +* Some internal memoryview functions were tuned to reduce object overhead. + +Bugs fixed +---------- + +* Exception position reporting could run into race conditions on threaded code. + It now uses function-local variables again. + +* Error handling early in the module init code could lead to a crash. + +* Error handling in ``cython.array`` creation was improved to avoid calling + C-API functions with an error held. + +* Complex buffer item types of structs of arrays could fail to validate. + Patch by Leo and smutch. (Github issue :issue:`1407`) + +* When importing the old Cython ``build_ext`` integration with distutils, the + additional command line arguments leaked into the regular command. + Patch by Kamekameha. (Github issue :issue:`2209`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could generate invalid C code. + (Github issue :issue:`3558`) + +* ``PyEval_InitThreads()`` is no longer used in Py3.7+ where it is a no-op. + +* Parallel builds of Cython itself (``setup.py build_ext -j N``) failed on Windows. + +Other changes +------------- + +* The C property feature has been rewritten and now requires C property methods + to be declared ``inline`` (:issue:`3571`). + + +3.0.0 alpha 3 (2020-04-27) +========================== + +Features added +-------------- + +* ``nogil`` functions now avoid acquiring the GIL on function exit if possible + even if they contain ``with gil`` blocks. + (Github issue :issue:`3554`) + +* Python private name mangling now falls back to unmangled names for non-Python + globals, since double-underscore names are not uncommon in C. Unmangled Python + names are also still found as a legacy fallback but produce a warning. + Patch by David Woods. (Github issue :issue:`3548`) + +Bugs fixed +---------- + +* Includes all bug-fixes from the :ref:`0.29.17` release. + + +3.0.0 alpha 2 (2020-04-23) +========================== + +Features added +-------------- + +* ``std::move()`` is now used in C++ mode for internal temp variables to + make them work without copying values. + Patch by David Woods. (Github issues :issue:`3253`, :issue:`1612`) + +* ``__class_getitem__`` is supported for types on item access (`PEP-560`_). + Patch by msg555. (Github issue :issue:`2753`) + +* The simplified Py3.6 customisation of class creation is implemented (`PEP-487`_). + (Github issue :issue:`2781`) + +* Conditional blocks in Python code that depend on ``cython.compiled`` are + eliminated at an earlier stage, which gives more freedom in writing + replacement Python code. + Patch by David Woods. (Github issue :issue:`3507`) + +* ``numpy.import_array()`` is automatically called if ``numpy`` has been cimported + and it has not been called in the module code. This is intended as a hidden + fail-safe so user code should continue to call ``numpy.import_array``. + Patch by David Woods. (Github issue :issue:`3524`) + +* The Cython AST code serialiser class ``CodeWriter`` in ``Cython.CodeWriter`` + supports more syntax nodes. + +* The fastcall/vectorcall protocols are used for several internal Python calls. + (Github issue :issue:`3540`) + +Bugs fixed +---------- + +* With ``language_level=3/3str``, Python classes without explicit base class + are now new-style (type) classes also in Py2. Previously, they were created + as old-style (non-type) classes. + (Github issue :issue:`3530`) + +* C++ ``typeid()`` failed for fused types. + Patch by David Woods. (Github issue :issue:`3203`) + +* ``__arg`` argument names in methods were not mangled with the class name. + Patch by David Woods. (Github issue :issue:`1382`) + +* Creating an empty unicode slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3531`) + +* Decoding an empty bytes/char* slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3534`) + +* Temporary buffer indexing variables were not released and could show up in + C compiler warnings, e.g. in generators. + Patch by David Woods. (Github issues :issue:`3430`, :issue:`3522`) + +* Several C compiler warnings were fixed. + + +3.0.0 alpha 1 (2020-04-12) +========================== + +Features added +-------------- + +* Cython functions now use the `PEP-590`_ vectorcall protocol in Py3.7+. + Patch by Jeroen Demeyer. (Github issue :issue:`2263`) + +* Unicode identifiers are supported in Cython code (`PEP-3131`_). + Patch by David Woods. (Github issue :issue:`2601`) + +* Unicode module names and imports are supported. + Patch by David Woods. (Github issue :issue:`3119`) + +* Annotations are no longer parsed, keeping them as strings following `PEP-563`_. + Patch by David Woods. (Github issue :issue:`3285`) + +* Preliminary support for the CPython's ``Py_LIMITED_API`` (stable ABI) is + available by setting the ``CYTHON_LIMITED_API`` C macro. Note that the + support is currently in an early stage and many features do not yet work. + You currently still have to define ``Py_LIMITED_API`` externally in order + to restrict the API usage. This will change when the feature stabilises. + Patches by Eddie Elizondo and David Woods. (Github issues :issue:`3223`, + :issue:`3311`, :issue:`3501`) + +* The dispatch to fused functions is now linear in the number of arguments, + which makes it much faster, often 2x or more, and several times faster for + larger fused types with many specialisations. + Patch by will-ca. (Github issue :issue:`1385`) + +* ``with gil/nogil`` statements can be conditional based on compile-time + constants, e.g. fused type checks. + Patch by Noam Hershtig. (Github issue :issue:`2579`) + +* ``const`` can be used together with fused types. + Patch by Thomas Vincent. (Github issue :issue:`1772`) + +* Reimports of already imported modules are substantially faster. + (Github issue :issue:`2854`) + +* Positional-only arguments are supported in Python functions (`PEP-570`_). + Patch by Josh Tobin. (Github issue :issue:`2915`) + +* The ``volatile`` C modifier is supported in Cython code. + Patch by Jeroen Demeyer. (Github issue :issue:`1667`) + +* ``@cython.trashcan(True)`` can be used on an extension type to enable the + CPython :ref:`trashcan`. This allows deallocating deeply recursive objects + without overflowing the stack. Patch by Jeroen Demeyer. (Github issue :issue:`2842`) + +* Inlined properties can be defined for external extension types. + Patch by Matti Picus. (Github issue :issue:`2640`, redone later in :issue:`3571`) + +* The ``str()`` builtin now calls ``PyObject_Str()`` instead of going + through a Python call. + Patch by William Ayd. (Github issue :issue:`3279`) + +* String concatenation can now happen in place if possible, by extending the + existing string rather than always creating a new one. + Patch by David Woods. (Github issue :issue:`3453`) + +* Multiplication of Python numbers with small constant integers is faster. + (Github issue :issue:`2808`) + +* Some list copying is avoided internally when a new list needs to be created + but we already have a fresh one. + (Github issue :issue:`3494`) + +* Extension types that do not need their own ``tp_new`` implementation (because + they have no object attributes etc.) directly inherit the implementation of + their parent type if possible. + (Github issue :issue:`1555`) + +* The attributes ``gen.gi_frame`` and ``coro.cr_frame`` of Cython compiled + generators and coroutines now return an actual frame object for introspection. + (Github issue :issue:`2306`) + +* Several declarations in ``cpython.*``, ``libc.*`` and ``libcpp.*`` were added. + Patches by Jeroen Demeyer, Matthew Edwards, Chris Gyurgyik, Jerome Kieffer + and Zackery Spytz. + (Github issues :issue:`3468`, :issue:`3332`, :issue:`3202`, :issue:`3188`, + :issue:`3179`, :issue:`2891`, :issue:`2826`, :issue:`2713`) + +* Deprecated NumPy API usages were removed from ``numpy.pxd``. + Patch by Matti Picus. (Github issue :issue:`3365`) + +* ``cython.inline()`` now sets the ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` + C macro automatically when ``numpy`` is imported in the code, to avoid C compiler + warnings about deprecated NumPy C-API usage. + +* The builtin ``abs()`` function can now be used on C numbers in nogil code. + Patch by Elliott Sales de Andrade. (Github issue :issue:`2748`) + +* `PEP-479`_ (``generator_stop``) is now enabled by default with language level 3. + (Github issue :issue:`2580`) + +* The ``cython.view.array`` type supports inheritance. + Patch by David Woods. (Github issue :issue:`3413`) + +* Code annotation accepts a new debugging argument ``--annotate-fullc`` that + will include the complete syntax highlighted C file in the HTML output. + (Github issue :issue:`2855`) + +* ``--no-capture`` added to ``runtests.py`` to prevent stdout/stderr capturing + during srctree tests. + Patch by Matti Picus. (Github issue :issue:`2701`) + +* ``--no-docstrings`` option added to ``cythonize`` script. + Original patch by mo-han. (Github issue :issue:`2889`) + +* ``cygdb`` gives better error messages when it fails to initialise the + Python runtime support in gdb. + Patch by Volker Weissmann. (Github issue :issue:`3489`) + +* The Pythran ``shape`` attribute is supported. + Patch by Serge Guelton. (Github issue :issue:`3307`) + +Bugs fixed +---------- + +* The unicode methods ``.upper()``, ``.lower()`` and ``.title()`` were + incorrectly optimised for single character input values and only returned + the first character if multiple characters should have been returned. + They now use the original Python methods again. + +* Fused argument types were not correctly handled in type annotations and + ``cython.locals()``. + Patch by David Woods. (Github issues :issue:`3391`, :issue:`3142`) + +* Diverging from the usual behaviour, ``len(memoryview)``, ``len(char*)`` + and ``len(Py_UNICODE*)`` returned an unsigned ``size_t`` value. They now + return a signed ``Py_ssize_t``, like other usages of ``len()``. + +* Nested dict literals in function call kwargs could incorrectly raise an + error about duplicate keyword arguments, which are allowed when passing + them from dict literals. + (Github issue :issue:`2963`) + +* Item access (subscripting) with integer indices/keys always tried the + Sequence protocol before the Mapping protocol, which diverged from Python + semantics. It now passes through the Mapping protocol first when supported. + (Github issue :issue:`1807`) + +* Name lookups in class bodies no longer go through an attribute lookup. + Patch by Jeroen Demeyer. (Github issue :issue:`3100`) + +* Broadcast assignments to a multi-dimensional memory view slice could end + up in the wrong places when the underlying memory view is known to be + contiguous but the slice is not. + (Github issue :issue:`2941`) + +* Pickling unbound methods of Python classes failed. + Patch by Pierre Glaser. (Github issue :issue:`2972`) + +* The ``Py_hash_t`` type failed to accept arbitrary "index" values. + (Github issue :issue:`2752`) + +* The first function line number of functions with decorators pointed to the + signature line and not the first decorator line, as in Python. + Patch by Felix Kohlgrüber. (Github issue :issue:`2536`) + +* Constant integer expressions that used a negative exponent were evaluated + as integer 0 instead of the expected float value. + Patch by Kryštof Pilnáček. (Github issue :issue:`2133`) + +* The ``cython.declare()`` and ``cython.cast()`` functions could fail in pure mode. + Patch by Dmitry Shesterkin. (Github issue :issue:`3244`) + +* ``__doc__`` was not available inside of the class body during class creation. + (Github issue :issue:`1635`) + +* Setting ``language_level=2`` in a file did not work if ``language_level=3`` + was enabled globally before. + Patch by Jeroen Demeyer. (Github issue :issue:`2791`) + +* ``__init__.pyx`` files were not always considered as package indicators. + (Github issue :issue:`2665`) + +* Compiling package ``__init__`` files could fail under Windows due to an + undefined export symbol. (Github issue :issue:`2968`) + +* A C compiler cast warning was resolved. + Patch by Michael Buesch. (Github issue :issue:`2775`) + +* Binding staticmethods of Cython functions were not behaving like Python methods. + Patch by Jeroen Demeyer. (Github issue :issue:`3106`, :issue:`3102`) + +* Memoryviews failed to compile when the ``cache_builtins`` feature was disabled. + Patch by David Woods. (Github issue :issue:`3406`) + +Other changes +------------- + +* The default language level was changed to ``3str``, i.e. Python 3 semantics, + but with ``str`` literals (also in Python 2.7). This is a backwards incompatible + change from the previous default of Python 2 semantics. The previous behaviour + is available through the directive ``language_level=2``. + (Github issue :issue:`2565`) + +* Cython no longer generates ``__qualname__`` attributes for classes in Python + 2.x since they are problematic there and not correctly maintained for subclasses. + Patch by Jeroen Demeyer. (Github issue :issue:`2772`) + +* Source file fingerprinting now uses SHA-1 instead of MD5 since the latter + tends to be slower and less widely supported these days. + (Github issue :issue:`2790`) + +* The long deprecated include files ``python_*``, ``stdio``, ``stdlib`` and + ``stl`` in ``Cython/Includes/Deprecated/`` were removed. Use the ``libc.*`` + and ``cpython.*`` pxd modules instead. + Patch by Jeroen Demeyer. (Github issue :issue:`2904`) + +* The search order for include files was changed. Previously it was + ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is + ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to + allow third-party ``*.pxd`` files to override the ones in Cython. + Patch by Matti Picus. (Github issue :issue:`2905`) + +* The command line parser was rewritten and modernised using ``argparse``. + Patch by Egor Dranischnikow. (Github issue :issue:`2952`, :issue:`3001`) + +* Dotted filenames for qualified module names (``pkg.mod.pyx``) are deprecated. + Use the normal Python package directory layout instead. + (Github issue :issue:`2686`) + +* Binary Linux wheels now follow the manylinux2010 standard. + Patch by Alexey Stepanov. (Github issue :issue:`3355`) + +* Support for Python 2.6 was removed. + +.. _`PEP-560`: https://www.python.org/dev/peps/pep-0560 +.. _`PEP-570`: https://www.python.org/dev/peps/pep-0570 +.. _`PEP-487`: https://www.python.org/dev/peps/pep-0487 +.. _`PEP-590`: https://www.python.org/dev/peps/pep-0590 +.. _`PEP-3131`: https://www.python.org/dev/peps/pep-3131 +.. _`PEP-563`: https://www.python.org/dev/peps/pep-0563 +.. _`PEP-479`: https://www.python.org/dev/peps/pep-0479 + + +.. _0.29.28: + +0.29.28 (2022-02-17) +==================== + +Bugs fixed +---------- + +* Due to backwards incompatible changes in CPython 3.11a4, the feature flags + ``CYTHON_FAST_THREAD_STATE`` and ``CYTHON_USE_EXC_INFO_STACK`` are now disabled + in Python 3.11 and later. They are enabled again in Cython 3.0. + Patch by David Woods. (Github issue #4610) + +* A C compiler warning in older PyPy versions was resolved. + Patch by Matti Picus. (Github issue #4236) + + +.. _0.29.27: + +0.29.27 (2022-01-28) +==================== + +Features added +-------------- + +* The ``cythonize`` command has a new option ``-M`` to generate ``.dep`` dependency + files for the compilation unit. This can be used by external build tools to track + these dependencies. + Patch by Evgeni Burovski. (Github issue :issue:`1214`) + +Bugs fixed +---------- + +* Compilation failures on PyPy were resolved. + Patches by Matti Picus. (Github issues :issue:`4509`, :issue:`4517`) + +* Calls to ``range()`` with more than three arguments did not fail. + Original patch by Max Bachmann. (Github issue :issue:`4550`) + +* Some C compiler warnings about missing type struct initialisers in Py3.10 were resolved. + +* Cython no longer warns about using OpenMP 3.0 features since they are now + considered generally available. + + +.. _0.29.26: + +0.29.26 (2021-12-16) +==================== + +Bugs fixed +---------- + +* An incompatibility with CPython 3.11.0a3 was resolved. + (Github issue :issue:`4499`) + +* The ``in`` operator failed on literal lists with starred expressions. + Patch by Arvind Natarajan. (Github issue :issue:`3938`) + +* A C compiler warning in PyPy about a missing struct field initialisation was resolved. + + +.. _0.29.25: + +0.29.25 (2021-12-06) +==================== + +Bugs fixed +---------- + +* Several incompatibilities with CPython 3.11 were resolved. + Patches by David Woods, Victor Stinner, Thomas Caswell. + (Github issues :issue:`4411`, :issue:`4414`, :issue:`4415`, :issue:`4416`, :issue:`4420`, + :issue:`4428`, :issue:`4473`, :issue:`4479`, :issue:`4480`) + +* Some C compiler warnings were resolved. + Patches by Lisandro Dalcin and others. (Github issue :issue:`4439`) + +* C++ ``std::move()`` should only be used automatically in MSVC versions that support it. + Patch by Max Bachmann. (Github issue :issue:`4191`) + + * The ``Py_hash_t`` type failed to accept arbitrary "index" values. + (Github issue :issue:`2752`) + +* Avoid copying unaligned 16-bit values since some platforms require them to be aligned. + Use memcpy() instead to let the C compiler decide how to do it. + (Github issue :issue:`4343`) + +* Cython crashed on invalid truthiness tests on C++ types without ``operator bool``. + Patch by David Woods. (Github issue :issue:`4348`) + +* The declaration of ``PyUnicode_CompareWithASCIIString()`` in ``cpython.unicode`` was incorrect. + Patch by Max Bachmann. (Github issue :issue:`4344`) + + +.. _0.29.24: + +0.29.24 (2021-07-14) +==================== + +Bugs fixed +---------- + +* Inline functions in pxd files that used memory views could lead to invalid + C code if the module that imported from them does not use memory views. + Patch by David Woods. (Github issue :issue:`1415`) + +* Several declarations in ``libcpp.string`` were added and corrected. + Patch by Janek Bevendorff. (Github issue :issue:`4268`) + +* Pickling unbound Cython compiled methods failed. + Patch by Pierre Glaser. (Github issue :issue:`2972`) + +* The tracing code was adapted to work with CPython 3.10. + +* The optimised ``in`` operator failed on unicode strings in Py3.9 and later + that were constructed from an external ``wchar_t`` source. + Also, related C compiler warnings about deprecated C-API usage were resolved. + (Github issue :issue:`3925`) + +* Some compiler crashes were resolved. + Patch by David Woods. (Github issues :issue:`4214`, :issue:`2811`) + +* An incorrect warning about 'unused' generator expressions was removed. + (GIthub issue :issue:`1699`) + +* The attributes ``gen.gi_frame`` and ``coro.cr_frame`` of Cython compiled + generators and coroutines now return an actual frame object for introspection, + instead of ``None``. + (Github issue :issue:`2306`) + + +.. _0.29.23: + +0.29.23 (2021-04-14) +==================== + +Bugs fixed +---------- + +* Some problems with Python 3.10 were resolved. + Patches by Victor Stinner and David Woods. (Github issues :issue:`4046`, :issue:`4100`) + +* An incorrect "optimisation" was removed that allowed changes to a keyword + dict to leak into keyword arguments passed into a function. + Patch by Peng Weikang. (Github issue :issue:`3227`) + +* Multiplied str constants could end up as bytes constants with language_level=2. + Patch by Alphadelta14 and David Woods. (Github issue :issue:`3951`) + +* ``PY_SSIZE_T_CLEAN`` does not get defined any more if it is already defined. + Patch by Andrew Jones. (Github issue :issue:`4104`) + + +.. _0.29.22: + +0.29.22 (2021-02-20) +==================== + +Features added +-------------- + +* Some declarations were added to the provided pxd includes. + Patches by Zackery Spytz and John Kirkham. + (Github issues :issue:`3811`, :issue:`3882`, :issue:`3899`, :issue:`3901`) + +Bugs fixed +---------- + +* A crash when calling certain functions in Py3.9 and later was resolved. + (Github issue :issue:`3917`) + +* ``const`` memory views of structs failed to compile. + (Github issue :issue:`2251`) + +* ``const`` template declarations could not be nested. + Patch by Ashwin Srinath. (Github issue :issue:`1355`) + +* The declarations in the ``cpython.pycapsule`` module were missing their + ``const`` modifiers and generated incorrect C code. + Patch by Warren Weckesser. (Github issue :issue:`3964`) + +* Casts to memory views failed for fused dtypes. + Patch by David Woods. (Github issue :issue:`3881`) + +* ``repr()`` was assumed to return ``str`` instead of ``unicode`` with ``language_level=3``. + (Github issue :issue:`3736`) + +* Calling ``cpdef`` functions from cimported modules crashed the compiler. + Patch by David Woods. (Github issue :issue:`4000`) + +* Cython no longer validates the ABI size of the NumPy classes it compiled against. + See the discussion in https://github.com/numpy/numpy/pull/432 + +* A C compiler warning about enum value casting was resolved in GCC. + (Github issue :issue:`2749`) + +* Coverage reporting in the annotated HTML file failed in Py3.9. + Patch by Nick Pope. (Github issue :issue:`3865`) + +* The embedding code now reports Python errors as exit status. + +* Long type declarations could lead to (harmless) random changes in the + C file when used in auto-generated Python wrappers or pickled classes. + +Other changes +------------- + +* Variables defined as ``cpdef`` now generate a warning since this + is currently useless and thus does not do what users would expect. + Patch by David Woods. (Github issue :issue:`3959`) + + +.. _0.29.21: + +0.29.21 (2020-07-09) +==================== + +Bugs fixed +---------- + +* Fix a regression in 0.29.20 where ``__div__`` failed to be found in extension types. + (Github issue :issue:`3688`) + +* Fix a regression in 0.29.20 where a call inside of a finally clause could fail to compile. + Patch by David Woods. (Github issue :issue:`3712`) + +* Zero-sized buffers could fail to validate as C/Fortran-contiguous. + Patch by Clemens Hofreither. (Github issue :issue:`2093`) + +* ``exec()`` did not allow recent Python syntax features in Py3.8+ due to + https://bugs.python.org/issue35975. + (Github issue :issue:`3695`) + +* Binding staticmethods of Cython functions were not behaving like Python methods in Py3. + Patch by Jeroen Demeyer and Michał Górny. (Github issue :issue:`3106`) + +* Pythran calls to NumPy methods no longer generate useless method lookup code. + +* The ``PyUnicode_GET_LENGTH()`` macro was missing from the ``cpython.*`` declarations. + Patch by Thomas Caswell. (Github issue :issue:`3692`) + +* The deprecated ``PyUnicode_*()`` C-API functions are no longer used, except for Unicode + strings that contain lone surrogates. Unicode strings that contain non-BMP characters + or surrogate pairs now generate different C code on 16-bit Python 2.x Unicode deployments + (such as MS-Windows). Generating the C code on Python 3.x is recommended in this case. + Original patches by Inada Naoki and Victor Stinner. + (Github issues :issue:`3677`, :issue:`3721`, :issue:`3697`) + +* Some template parameters were missing from the C++ ``std::unordered_map`` declaration. + Patch by will. (Github issue :issue:`3685`) + +* Several internal code generation issues regarding temporary variables were resolved. + (Github issue :issue:`3708`) + + +.. _0.29.20: + +0.29.20 (2020-06-10) +==================== + +Bugs fixed +---------- + +* Nested try-except statements with multiple ``return`` statements could crash + due to incorrect deletion of the ``except as`` target variable. + (Github issue :issue:`3666`) + +* The ``@classmethod`` decorator no longer rejects unknown input from other decorators. + Patch by David Woods. (Github issue :issue:`3660`) + +* Fused types could leak into unrelated usages. + Patch by David Woods. (Github issue :issue:`3642`) + +* Now uses ``Py_SET_SIZE()`` and ``Py_SET_REFCNT()`` in Py3.9+ to avoid low-level + write access to these object fields. + Patch by Victor Stinner. (Github issue :issue:`3639`) + +* The built-in ``abs()`` function could lead to undefined behaviour when used on + the negative-most value of a signed C integer type. + Patch by Serge Guelton. (Github issue :issue:`1911`) + +* Usages of ``sizeof()`` and ``typeid()`` on uninitialised variables no longer + produce a warning. + Patch by Celelibi. (Github issue :issue:`3575`) + +* The C++ ``typeid()`` function was allowed in C mode. + Patch by Celelibi. (Github issue :issue:`3637`) + +* The error position reported for errors found in f-strings was misleading. + (Github issue :issue:`3674`) + +* The new ``c_api_binop_methods`` directive was added for forward compatibility, but can + only be set to True (the current default value). It can be disabled in Cython 3.0. + + +.. _0.29.19: + +0.29.19 (2020-05-20) +==================== + +Bugs fixed +---------- + +* A typo in Windows specific code in 0.29.18 was fixed that broke "libc.math". + (Github issue :issue:`3622`) + +* A platform specific test failure in 0.29.18 was fixed. + Patch by smutch. (Github issue :issue:`3620`) + + +.. _0.29.18: + +0.29.18 (2020-05-18) +==================== + +Bugs fixed +---------- + +* Exception position reporting could run into race conditions on threaded code. + It now uses function-local variables again. + +* Error handling early in the module init code could lead to a crash. + +* Error handling in ``cython.array`` creation was improved to avoid calling + C-API functions with an error held. + +* A memory corruption was fixed when garbage collection was triggered during calls + to ``PyType_Ready()`` of extension type subclasses. + (Github issue :issue:`3603`) + +* Memory view slicing generated unused error handling code which could negatively + impact the C compiler optimisations for parallel OpenMP code etc. Also, it is + now helped by static branch hints. + (Github issue :issue:`2987`) + +* Cython's built-in OpenMP functions were not translated inside of call arguments. + Original patch by Celelibi and David Woods. (Github issue :issue:`3594`) + +* Complex buffer item types of structs of arrays could fail to validate. + Patch by Leo and smutch. (Github issue :issue:`1407`) + +* Decorators were not allowed on nested `async def` functions. + (Github issue :issue:`1462`) + +* C-tuples could use invalid C struct casting. + Patch by MegaIng. (Github issue :issue:`3038`) + +* Optimised ``%d`` string formatting into f-strings failed on float values. + (Github issue :issue:`3092`) + +* Optimised aligned string formatting (``%05s``, ``%-5s``) failed. + (Github issue :issue:`3476`) + +* When importing the old Cython ``build_ext`` integration with distutils, the + additional command line arguments leaked into the regular command. + Patch by Kamekameha. (Github issue :issue:`2209`) + +* When using the ``CYTHON_NO_PYINIT_EXPORT`` option in C++, the module init function + was not declared as ``extern "C"``. + (Github issue :issue:`3414`) + +* Three missing timedelta access macros were added in ``cpython.datetime``. + +* The signature of the NumPy C-API function ``PyArray_SearchSorted()`` was fixed. + Patch by Brock Mendel. (Github issue :issue:`3606`) + + +.. _0.29.17: + +0.29.17 (2020-04-26) +==================== + +Features added +-------------- + +* ``std::move()`` is now available from ``libcpp.utility``. + Patch by Omer Ozarslan. (Github issue :issue:`2169`) + +* The ``@cython.binding`` decorator is available in Python code. + (Github issue :issue:`3505`) + +Bugs fixed +---------- + +* Creating an empty unicode slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3531`) + +* Decoding an empty bytes/char* slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3534`) + +* Re-importing a Cython extension no longer raises the error + "``__reduce_cython__ not found``". + (Github issue :issue:`3545`) + +* Unused C-tuples could generate incorrect code in 0.29.16. + Patch by Kirk Meyer. (Github issue :issue:`3543`) + +* Creating a fused function attached it to the garbage collector before it + was fully initialised, thus risking crashes in rare failure cases. + Original patch by achernomorov. (Github issue :issue:`3215`) + +* Temporary buffer indexing variables were not released and could show up in + C compiler warnings, e.g. in generators. + Patch by David Woods. (Github issues :issue:`3430`, :issue:`3522`) + +* The compilation cache in ``cython.inline("…")`` failed to take the language + level into account. + Patch by will-ca. (Github issue :issue:`3419`) + +* The deprecated ``PyUnicode_GET_SIZE()`` function is no longer used in Py3. + + +.. _0.29.16: + +0.29.16 (2020-03-24) +==================== + +Bugs fixed +---------- + +* Temporary internal variables in nested prange loops could leak into other + threads. Patch by Frank Schlimbach. (Github issue :issue:`3348`) + +* Default arguments on fused functions could crash. + Patch by David Woods. (Github issue :issue:`3370`) + +* C-tuples declared in ``.pxd`` files could generate incomplete C code. + Patch by Kirk Meyer. (Github issue :issue:`1427`) + +* Fused functions were not always detected and optimised as Cython + implemented functions. + Patch by David Woods. (Github issue :issue:`3384`) + +* Valid Python object concatenation of (iterable) strings to non-strings + could fail with an exception. + Patch by David Woods. (Github issue :issue:`3433`) + +* Using C functions as temporary values lead to invalid C code. + Original patch by David Woods. (Github issue :issue:`3418`) + +* Fix an unhandled C++ exception in comparisons. + Patch by David Woods. (Github issue :issue:`3361`) + +* Fix deprecated import of "imp" module. + Patch by Matti Picus. (Github issue :issue:`3350`) + +* Fix compatibility with Pythran 0.9.6 and later. + Patch by Serge Guelton. (Github issue :issue:`3308`) + +* The ``_Py_PyAtExit()`` function in ``cpython.pylifecycle`` was misdeclared. + Patch by Zackery Spytz. (Github issue :issue:`3382`) + +* Several missing declarations in ``cpython.*`` were added. + Patches by Zackery Spytz. (Github issue :issue:`3452`, :issue:`3421`, :issue:`3411`, :issue:`3402`) + +* A declaration for ``libc.math.fpclassify()`` was added. + Patch by Zackery Spytz. (Github issue :issue:`2514`) + +* Avoid "undeclared" warning about automatically generated pickle methods. + Patch by David Woods. (Github issue :issue:`3353`) + +* Avoid C compiler warning about unreachable code in ``prange()``. + +* Some C compiler warnings in PyPy were resolved. + Patch by Matti Picus. (Github issue :issue:`3437`) + + +.. _0.29.15: + + +0.29.15 (2020-02-06) +==================== + +Bugs fixed +---------- + +* Crash when returning a temporary Python object from an async-def function. + (Github issue :issue:`3337`) + +* Crash when using ``**kwargs`` in generators. + Patch by David Woods. (Github issue :issue:`3265`) + +* Double reference free in ``__class__`` cell handling for ``super()`` calls. + (Github issue :issue:`3246`) + +* Compile error when using ``*args`` as Python class bases. + (Github issue :issue:`3338`) + +* Import failure in IPython 7.11. + (Github issue :issue:`3297`) + +* Fixed C name collision in the auto-pickle code. + Patch by ThePrez. (Github issue :issue:`3238`) + +* Deprecated import failed in Python 3.9. + (Github issue :issue:`3266`) + + +.. _0.29.14: + +0.29.14 (2019-11-01) +==================== + +Bugs fixed +---------- + +* The generated code failed to initialise the ``tp_print`` slot in CPython 3.8. + Patches by Pablo Galindo and Orivej Desh. (Github issues :issue:`3171`, :issue:`3201`) + +* ``?`` for ``bool`` was missing from the supported NumPy dtypes. + Patch by Max Klein. (Github issue :issue:`2675`) + +* ``await`` was not allowed inside of f-strings. + Patch by Dmitro Getz. (Github issue :issue:`2877`) + +* Coverage analysis failed for projects where the code resides in separate + source sub-directories. + Patch by Antonio Valentino. (Github issue :issue:`1985`) + +* An incorrect compiler warning was fixed in automatic C++ string conversions. + Patch by Gerion Entrup. (Github issue :issue:`3108`) + +* Error reports in the Jupyter notebook showed unhelpful stack traces. + Patch by Matthew Edwards (Github issue :issue:`3196`). + +* ``Python.h`` is now also included explicitly from ``public`` header files. + (Github issue :issue:`3133`). + +* Distutils builds with ``--parallel`` did not work when using Cython's + deprecated ``build_ext`` command. + Patch by Alphadelta14 (Github issue :issue:`3187`). + +Other changes +------------- + +* The ``PyMemoryView_*()`` C-API is available in ``cpython.memoryview``. + Patch by Nathan Manville. (Github issue :issue:`2541`) + + +0.29.13 (2019-07-26) +==================== + +Bugs fixed +---------- + +* A reference leak for ``None`` was fixed when converting a memoryview + to a Python object. (Github issue :issue:`3023`) + +* The declaration of ``PyGILState_STATE`` in ``cpython.pystate`` was unusable. + Patch by Kirill Smelkov. (Github issue :issue:`2997`) + +Other changes +------------- + +* The declarations in ``posix.mman`` were extended. + Patches by Kirill Smelkov. (Github issues :issue:`2893`, :issue:`2894`, :issue:`3012`) + + +0.29.12 (2019-07-07) +==================== + +Bugs fixed +---------- + +* Fix compile error in CPython 3.8b2 regarding the ``PyCode_New()`` signature. + (Github issue :issue:`3031`) + +* Fix a C compiler warning about a missing ``int`` downcast. + (Github issue :issue:`3028`) + +* Fix reported error positions of undefined builtins and constants. + Patch by Orivej Desh. (Github issue :issue:`3030`) + +* A 32 bit issue in the Pythran support was resolved. + Patch by Serge Guelton. (Github issue :issue:`3032`) + + +0.29.11 (2019-06-30) +==================== + +Bugs fixed +---------- + +* Fix compile error in CPython 3.8b2 regarding the ``PyCode_New()`` signature. + Patch by Nick Coghlan. (Github issue :issue:`3009`) + +* Invalid C code generated for lambda functions in cdef methods. + Patch by Josh Tobin. (Github issue :issue:`2967`) + +* Support slice handling in newer Pythran versions. + Patch by Serge Guelton. (Github issue :issue:`2989`) + +* A reference leak in power-of-2 calculation was fixed. + Patch by Sebastian Berg. (Github issue :issue:`3022`) + +* The search order for include files was changed. Previously it was + ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is + ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to + allow third-party ``*.pxd`` files to override the ones in Cython. + Original patch by Matti Picus. (Github issue :issue:`2905`) + +* Setting ``language_level=2`` in a file did not work if ``language_level=3`` + was enabled globally before. + Patch by Jeroen Demeyer. (Github issue :issue:`2791`) + + +0.29.10 (2019-06-02) +==================== + +Bugs fixed +---------- + +* Fix compile errors in CPython 3.8b1 due to the new "tp_vectorcall" slots. + (Github issue :issue:`2976`) + + +0.29.9 (2019-05-29) +=================== + +Bugs fixed +---------- + +* Fix a crash regression in 0.29.8 when creating code objects fails. + +* Remove an incorrect cast when using true-division in C++ operations. + (Github issue :issue:`1950`) + + +0.29.8 (2019-05-28) +=================== + +Bugs fixed +---------- + +* C compile errors with CPython 3.8 were resolved. + Patch by Marcel Plch. (Github issue :issue:`2938`) + +* Python tuple constants that compare equal but have different item + types could incorrectly be merged into a single constant. + (Github issue :issue:`2919`) + +* Non-ASCII characters in unprefixed strings could crash the compiler when + used with language level ``3str``. + +* Starred expressions in %-formatting tuples could fail to compile for + unicode strings. (Github issue :issue:`2939`) + +* Passing Python class references through ``cython.inline()`` was broken. + (Github issue :issue:`2936`) + + +0.29.7 (2019-04-14) +=================== + +Bugs fixed +---------- + +* Crash when the shared Cython config module gets unloaded and another Cython + module reports an exceptions. Cython now makes sure it keeps an owned reference + to the module. + (Github issue :issue:`2885`) + +* Resolved a C89 compilation problem when enabling the fast-gil sharing feature. + +* Coverage reporting did not include the signature line of ``cdef`` functions. + (Github issue :issue:`1461`) + +* Casting a GIL-requiring function into a nogil function now issues a warning. + (Github issue :issue:`2879`) + +* Generators and coroutines were missing their return type annotation. + (Github issue :issue:`2884`) + + +0.29.6 (2019-02-27) +=================== + +Bugs fixed +---------- + +* Fix a crash when accessing the ``__kwdefaults__`` special attribute of + fused functions. (Github issue :issue:`1470`) + +* Fix the parsing of buffer format strings that contain numeric sizes, which + could lead to incorrect input rejections. (Github issue :issue:`2845`) + +* Avoid a C #pragma in old gcc versions that was only added in GCC 4.6. + Patch by Michael Anselmi. (Github issue :issue:`2838`) + +* Auto-encoding of Unicode strings to UTF-8 C/C++ strings failed in Python 3, + even though the default encoding there is UTF-8. + (Github issue :issue:`2819`) + + +0.29.5 (2019-02-09) +=================== + +Bugs fixed +---------- + +* Crash when defining a Python subclass of an extension type and repeatedly calling + a cpdef method on it. (Github issue :issue:`2823`) + +* Compiler crash when ``prange()`` loops appear inside of with-statements. + (Github issue :issue:`2780`) + +* Some C compiler warnings were resolved. + Patches by Christoph Gohlke. (Github issues :issue:`2815`, :issue:`2816`, :issue:`2817`, :issue:`2822`) + +* Python conversion of C++ enums failed in 0.29. + Patch by Orivej Desh. (Github issue :issue:`2767`) + + +0.29.4 (2019-02-01) +=================== + +Bugs fixed +---------- + +* Division of numeric constants by a runtime value of 0 could fail to raise a + ``ZeroDivisionError``. (Github issue :issue:`2820`) + + +0.29.3 (2019-01-19) +=================== + +Bugs fixed +---------- + +* Some C code for memoryviews was generated in a non-deterministic order. + Patch by Martijn van Steenbergen. (Github issue :issue:`2779`) + +* C89 compatibility was accidentally lost since 0.28. + Patches by gastineau and true-pasky. (Github issues :issue:`2778`, :issue:`2801`) + +* A C compiler cast warning was resolved. + Patch by Michael Buesch. (Github issue :issue:`2774`) + +* An compilation failure with complex numbers under MSVC++ was resolved. + (Github issue :issue:`2797`) + +* Coverage reporting could fail when modules were moved around after the build. + Patch by Wenjun Si. (Github issue :issue:`2776`) + + +0.29.2 (2018-12-14) +=================== + +Bugs fixed +---------- + +* The code generated for deduplicated constants leaked some references. + (Github issue :issue:`2750`) + +* The declaration of ``sigismember()`` in ``libc.signal`` was corrected. + (Github issue :issue:`2756`) + +* Crashes in compiler and test runner were fixed. + (Github issue :issue:`2736`, :issue:`2755`) + +* A C compiler warning about an invalid safety check was resolved. + (Github issue :issue:`2731`) + + +0.29.1 (2018-11-24) +=================== + +Bugs fixed +---------- + +* Extensions compiled with MinGW-64 under Windows could misinterpret integer + objects larger than 15 bit and return incorrect results. + (Github issue :issue:`2670`) + +* Cython no longer requires the source to be writable when copying its data + into a memory view slice. + Patch by Andrey Paramonov. (Github issue :issue:`2644`) + +* Line tracing of ``try``-statements generated invalid C code. + (Github issue :issue:`2274`) + +* When using the ``warn.undeclared`` directive, Cython's own code generated + warnings that are now fixed. + Patch by Nicolas Pauss. (Github issue :issue:`2685`) + +* Cython's memoryviews no longer require strides for setting the shape field + but only the ``PyBUF_ND`` flag to be set. + Patch by John Kirkham. (Github issue :issue:`2716`) + +* Some C compiler warnings about unused memoryview code were fixed. + Patch by Ho Cheuk Ting. (Github issue :issue:`2588`) + +* A C compiler warning about implicit signed/unsigned conversion was fixed. + (Github issue :issue:`2729`) + +* Assignments to C++ references returned by ``operator[]`` could fail to compile. + (Github issue :issue:`2671`) + +* The power operator and the support for NumPy math functions were fixed + in Pythran expressions. + Patch by Serge Guelton. (Github issues :issue:`2702`, :issue:`2709`) + +* Signatures with memory view arguments now show the expected type + when embedded in docstrings. + Patch by Matthew Chan and Benjamin Weigel. (Github issue :issue:`2634`) + +* Some ``from ... cimport ...`` constructs were not correctly considered + when searching modified dependencies in ``cythonize()`` to decide + whether to recompile a module. + Patch by Kryštof Pilnáček. (Github issue :issue:`2638`) + +* A struct field type in the ``cpython.array`` declarations was corrected. + Patch by John Kirkham. (Github issue :issue:`2712`) + + +0.29 (2018-10-14) +================= + +Features added +-------------- + +* PEP-489 multi-phase module initialisation has been enabled again. Module + reloads in other subinterpreters raise an exception to prevent corruption + of the static module state. + +* A set of ``mypy`` compatible PEP-484 declarations were added for Cython's C data + types to integrate with static analysers in typed Python code. They are available + in the ``Cython/Shadow.pyi`` module and describe the types in the special ``cython`` + module that can be used for typing in Python code. + Original patch by Julian Gethmann. (Github issue :issue:`1965`) + +* Memoryviews are supported in PEP-484/526 style type declarations. + (Github issue :issue:`2529`) + +* ``@cython.nogil`` is supported as a C-function decorator in Python code. + (Github issue :issue:`2557`) + +* Raising exceptions from nogil code will automatically acquire the GIL, instead + of requiring an explicit ``with gil`` block. + +* C++ functions can now be declared as potentially raising both C++ and Python + exceptions, so that Cython can handle both correctly. + (Github issue :issue:`2615`) + +* ``cython.inline()`` supports a direct ``language_level`` keyword argument that + was previously only available via a directive. + +* A new language level name ``3str`` was added that mostly corresponds to language + level 3, but keeps unprefixed string literals as type 'str' in both Py2 and Py3, + and the builtin 'str' type unchanged. This will become the default in the next + Cython release and is meant to help user code a) transition more easily to this + new default and b) migrate to Python 3 source code semantics without making support + for Python 2.x difficult. + +* In CPython 3.6 and later, looking up globals in the module dict is almost + as fast as looking up C globals. + (Github issue :issue:`2313`) + +* For a Python subclass of an extension type, repeated method calls to non-overridden + cpdef methods can avoid the attribute lookup in Py3.6+, which makes them 4x faster. + (Github issue :issue:`2313`) + +* (In-)equality comparisons of objects to integer literals are faster. + (Github issue :issue:`2188`) + +* Some internal and 1-argument method calls are faster. + +* Modules that cimport many external extension types from other Cython modules + execute less import requests during module initialisation. + +* Constant tuples and slices are deduplicated and only created once per module. + (Github issue :issue:`2292`) + +* The coverage plugin considers more C file extensions such as ``.cc`` and ``.cxx``. + (Github issue :issue:`2266`) + +* The ``cythonize`` command accepts compile time variable values (as set by ``DEF``) + through the new ``-E`` option. + Patch by Jerome Kieffer. (Github issue :issue:`2315`) + +* ``pyximport`` can import from namespace packages. + Patch by Prakhar Goel. (Github issue :issue:`2294`) + +* Some missing numpy and CPython C-API declarations were added. + Patch by John Kirkham. (Github issues :issue:`2523`, :issue:`2520`, :issue:`2537`) + +* Declarations for the ``pylifecycle`` C-API functions were added in a new .pxd file + ``cpython.pylifecycle``. + +* The Pythran support was updated to work with the latest Pythran 0.8.7. + Original patch by Adrien Guinet. (Github issue :issue:`2600`) + +* ``%a`` is included in the string formatting types that are optimised into f-strings. + In this case, it is also automatically mapped to ``%r`` in Python 2.x. + +* New C macro ``CYTHON_HEX_VERSION`` to access Cython's version in the same style as + ``PY_VERSION_HEX``. + +* Constants in ``libc.math`` are now declared as ``const`` to simplify their handling. + +* An additional ``check_size`` clause was added to the ``ctypedef class`` name + specification to allow suppressing warnings when importing modules with + backwards-compatible ``PyTypeObject`` size changes. + Patch by Matti Picus. (Github issue :issue:`2627`) + +Bugs fixed +---------- + +* The exception handling in generators and coroutines under CPython 3.7 was adapted + to the newly introduced exception stack. Users of Cython 0.28 who want to support + Python 3.7 are encouraged to upgrade to 0.29 to avoid potentially incorrect error + reporting and tracebacks. (Github issue :issue:`1958`) + +* Crash when importing a module under Stackless Python that was built for CPython. + Patch by Anselm Kruis. (Github issue :issue:`2534`) + +* 2-value slicing of typed sequences failed if the start or stop index was None. + Patch by Christian Gibson. (Github issue :issue:`2508`) + +* Multiplied string literals lost their factor when they are part of another + constant expression (e.g. 'x' * 10 + 'y' => 'xy'). + +* String formatting with the '%' operator didn't call the special ``__rmod__()`` + method if the right side is a string subclass that implements it. + (Python issue 28598) + +* The directive ``language_level=3`` did not apply to the first token in the + source file. (Github issue :issue:`2230`) + +* Overriding cpdef methods did not work in Python subclasses with slots. + Note that this can have a performance impact on calls from Cython code. + (Github issue :issue:`1771`) + +* Fix declarations of builtin or C types using strings in pure python mode. + (Github issue :issue:`2046`) + +* Generator expressions and lambdas failed to compile in ``@cfunc`` functions. + (Github issue :issue:`459`) + +* Global names with ``const`` types were not excluded from star-import assignments + which could lead to invalid C code. + (Github issue :issue:`2621`) + +* Several internal function signatures were fixed that lead to warnings in gcc-8. + (Github issue :issue:`2363`) + +* The numpy helper functions ``set_array_base()`` and ``get_array_base()`` + were adapted to the current numpy C-API recommendations. + Patch by Matti Picus. (Github issue :issue:`2528`) + +* Some NumPy related code was updated to avoid deprecated API usage. + Original patch by jbrockmendel. (Github issue :issue:`2559`) + +* Several C++ STL declarations were extended and corrected. + Patch by Valentin Valls. (Github issue :issue:`2207`) + +* C lines of the module init function were unconditionally not reported in + exception stack traces. + Patch by Jeroen Demeyer. (Github issue :issue:`2492`) + +* When PEP-489 support is enabled, reloading the module overwrote any static + module state. It now raises an exception instead, given that reloading is + not actually supported. + +* Object-returning, C++ exception throwing functions were not checking that + the return value was non-null. + Original patch by Matt Wozniski (Github issue :issue:`2603`) + +* The source file encoding detection could get confused if the + ``c_string_encoding`` directive appeared within the first two lines. + (Github issue :issue:`2632`) + +* Cython generated modules no longer emit a warning during import when the + size of the NumPy array type is larger than what was found at compile time. + Instead, this is assumed to be a backwards compatible change on NumPy side. + +Other changes +------------- + +* Cython now emits a warning when no ``language_level`` (2, 3 or '3str') is set + explicitly, neither as a ``cythonize()`` option nor as a compiler directive. + This is meant to prepare the transition of the default language level from + currently Py2 to Py3, since that is what most new users will expect these days. + The future default will, however, not enforce unicode literals, because this + has proven a major obstacle in the support for both Python 2.x and 3.x. The + next major release is intended to make this change, so that it will parse all + code that does not request a specific language level as Python 3 code, but with + ``str`` literals. The language level 2 will continue to be supported for an + indefinite time. + +* The documentation was restructured, cleaned up and examples are now tested. + The NumPy tutorial was also rewritten to simplify the running example. + Contributed by Gabriel de Marmiesse. (Github issue :issue:`2245`) + +* Cython compiles less of its own modules at build time to reduce the installed + package size to about half of its previous size. This makes the compiler + slightly slower, by about 5-7%. + + +0.28.6 (2018-11-01) +=================== + +Bugs fixed +---------- + +* Extensions compiled with MinGW-64 under Windows could misinterpret integer + objects larger than 15 bit and return incorrect results. + (Github issue :issue:`2670`) + +* Multiplied string literals lost their factor when they are part of another + constant expression (e.g. 'x' * 10 + 'y' => 'xy'). + + +0.28.5 (2018-08-03) +=================== + +Bugs fixed +---------- + +* The discouraged usage of GCC's attribute ``optimize("Os")`` was replaced by the + similar attribute ``cold`` to reduce the code impact of the module init functions. + (Github issue :issue:`2494`) + +* A reference leak in Py2.x was fixed when comparing str to unicode for equality. + + +0.28.4 (2018-07-08) +=================== + +Bugs fixed +---------- + +* Reallowing ``tp_clear()`` in a subtype of an ``@no_gc_clear`` extension type + generated an invalid C function call to the (non-existent) base type implementation. + (Github issue :issue:`2309`) + +* Exception catching based on a non-literal (runtime) tuple could fail to match the + exception. (Github issue :issue:`2425`) + +* Compile fix for CPython 3.7.0a2. (Github issue :issue:`2477`) + + +0.28.3 (2018-05-27) +=================== + +Bugs fixed +---------- + +* Set iteration was broken in non-CPython since 0.28. + +* ``UnicodeEncodeError`` in Py2 when ``%s`` formatting is optimised for + unicode strings. (Github issue :issue:`2276`) + +* Work around a crash bug in g++ 4.4.x by disabling the size reduction setting + of the module init function in this version. (Github issue :issue:`2235`) + +* Crash when exceptions occur early during module initialisation. + (Github issue :issue:`2199`) + + +0.28.2 (2018-04-13) +=================== + +Features added +-------------- + +* ``abs()`` is faster for Python long objects. + +* The C++11 methods ``front()`` and ``end()`` were added to the declaration of + ``libcpp.string``. Patch by Alex Huszagh. (Github issue :issue:`2123`) + +* The C++11 methods ``reserve()`` and ``bucket_count()`` are declared for + ``libcpp.unordered_map``. Patch by Valentin Valls. (Github issue :issue:`2168`) + +Bugs fixed +---------- + +* The copy of a read-only memoryview was considered read-only as well, whereas + a common reason to copy a read-only view is to make it writable. The result + of the copying is now a writable buffer by default. + (Github issue :issue:`2134`) + +* The ``switch`` statement generation failed to apply recursively to the body of + converted if-statements. + +* ``NULL`` was sometimes rejected as exception return value when the returned + type is a fused pointer type. + Patch by Callie LeFave. (Github issue :issue:`2177`) + +* Fixed compatibility with PyPy 5.11. + Patch by Matti Picus. (Github issue :issue:`2165`) + +Other changes +------------- + +* The NumPy tutorial was rewritten to use memoryviews instead of the older + buffer declaration syntax. + Contributed by Gabriel de Marmiesse. (Github issue :issue:`2162`) + + +0.28.1 (2018-03-18) +=================== + +Bugs fixed +---------- + +* ``PyFrozenSet_New()`` was accidentally used in PyPy where it is missing + from the C-API. + +* Assignment between some C++ templated types were incorrectly rejected + when the templates mix ``const`` with ``ctypedef``. + (Github issue :issue:`2148`) + +* Undeclared C++ no-args constructors in subclasses could make the compilation + fail if the base class constructor was declared without ``nogil``. + (Github issue :issue:`2157`) + +* Bytes %-formatting inferred ``basestring`` (bytes or unicode) as result type + in some cases where ``bytes`` would have been safe to infer. + (Github issue :issue:`2153`) + +* ``None`` was accidentally disallowed as typed return value of ``dict.pop()``. + (Github issue :issue:`2152`) + + +0.28 (2018-03-13) +================= + +Features added +-------------- + +* Cdef classes can now multiply inherit from ordinary Python classes. + (The primary base must still be a c class, possibly ``object``, and + the other bases must *not* be cdef classes.) + +* Type inference is now supported for Pythran compiled NumPy expressions. + Patch by Nils Braun. (Github issue :issue:`1954`) + +* The ``const`` modifier can be applied to memoryview declarations to allow + read-only buffers as input. (Github issues :issue:`1605`, :issue:`1869`) + +* C code in the docstring of a ``cdef extern`` block is copied verbatimly + into the generated file. + Patch by Jeroen Demeyer. (Github issue :issue:`1915`) + +* When compiling with gcc, the module init function is now tuned for small + code size instead of whatever compile flags were provided externally. + Cython now also disables some code intensive optimisations in that function + to further reduce the code size. (Github issue :issue:`2102`) + +* Decorating an async coroutine with ``@cython.iterable_coroutine`` changes its + type at compile time to make it iterable. While this is not strictly in line + with PEP-492, it improves the interoperability with old-style coroutines that + use ``yield from`` instead of ``await``. + +* The IPython magic has preliminary support for JupyterLab. + (Github issue :issue:`1775`) + +* The new TSS C-API in CPython 3.7 is supported and has been backported. + Patch by Naotoshi Seo. (Github issue :issue:`1932`) + +* Cython knows the new ``Py_tss_t`` type defined in PEP-539 and automatically + initialises variables declared with that type to ``Py_tss_NEEDS_INIT``, + a value which cannot be used outside of static assignments. + +* The set methods ``.remove()`` and ``.discard()`` are optimised. + Patch by Antoine Pitrou. (Github issue :issue:`2042`) + +* ``dict.pop()`` is optimised. + Original patch by Antoine Pitrou. (Github issue :issue:`2047`) + +* Iteration over sets and frozensets is optimised. + (Github issue :issue:`2048`) + +* Safe integer loops (< range(2^30)) are automatically optimised into C loops. + +* ``alist.extend([a,b,c])`` is optimised into sequential ``list.append()`` calls + for short literal sequences. + +* Calls to builtin methods that are not specifically optimised into C-API calls + now use a cache that avoids repeated lookups of the underlying C function. + (Github issue :issue:`2054`) + +* Single argument function calls can avoid the argument tuple creation in some cases. + +* Some redundant extension type checks are avoided. + +* Formatting C enum values in f-strings is faster, as well as some other special cases. + +* String formatting with the '%' operator is optimised into f-strings in simple cases. + +* Subscripting (item access) is faster in some cases. + +* Some ``bytearray`` operations have been optimised similar to ``bytes``. + +* Some PEP-484/526 container type declarations are now considered for + loop optimisations. + +* Indexing into memoryview slices with ``view[i][j]`` is now optimised into + ``view[i, j]``. + +* Python compatible ``cython.*`` types can now be mixed with type declarations + in Cython syntax. + +* Name lookups in the module and in classes are faster. + +* Python attribute lookups on extension types without instance dict are faster. + +* Some missing signals were added to ``libc/signal.pxd``. + Patch by Jeroen Demeyer. (Github issue :issue:`1914`) + +* The warning about repeated extern declarations is now visible by default. + (Github issue :issue:`1874`) + +* The exception handling of the function types used by CPython's type slot + functions was corrected to match the de-facto standard behaviour, so that + code that uses them directly benefits from automatic and correct exception + propagation. Patch by Jeroen Demeyer. (Github issue :issue:`1980`) + +* Defining the macro ``CYTHON_NO_PYINIT_EXPORT`` will prevent the module init + function from being exported as symbol, e.g. when linking modules statically + in an embedding setup. Patch by AraHaan. (Github issue :issue:`1944`) + +Bugs fixed +---------- + +* If a module name is explicitly provided for an ``Extension()`` that is compiled + via ``cythonize()``, it was previously ignored and replaced by the source file + name. It can now be used to override the target module name, e.g. for compiling + prefixed accelerator modules from Python files. (Github issue :issue:`2038`) + +* The arguments of the ``num_threads`` parameter of parallel sections + were not sufficiently validated and could lead to invalid C code. + (Github issue :issue:`1957`) + +* Catching exceptions with a non-trivial exception pattern could call into + CPython with a live exception set. This triggered incorrect behaviour + and crashes, especially in CPython 3.7. + +* The signature of the special ``__richcmp__()`` method was corrected to recognise + the type of the first argument as ``self``. It was previously treated as plain + object, but CPython actually guarantees that it always has the correct type. + Note: this can change the semantics of user code that previously relied on + ``self`` being untyped. + +* Some Python 3 exceptions were not recognised as builtins when running Cython + under Python 2. + +* Some async helper functions were not defined in the generated C code when + compiling simple async code. (Github issue :issue:`2075`) + +* Line tracing did not include generators and coroutines. + (Github issue :issue:`1949`) + +* C++ declarations for ``unordered_map`` were corrected. + Patch by Michael Schatzow. (Github issue :issue:`1484`) + +* Iterator declarations in C++ ``deque`` and ``vector`` were corrected. + Patch by Alex Huszagh. (Github issue :issue:`1870`) + +* The const modifiers in the C++ ``string`` declarations were corrected, together + with the coercion behaviour of string literals into C++ strings. + (Github issue :issue:`2132`) + +* Some declaration types in ``libc.limits`` were corrected. + Patch by Jeroen Demeyer. (Github issue :issue:`2016`) + +* ``@cython.final`` was not accepted on Python classes with an ``@cython.cclass`` + decorator. (Github issue :issue:`2040`) + +* Cython no longer creates useless and incorrect ``PyInstanceMethod`` wrappers for + methods in Python 3. Patch by Jeroen Demeyer. (Github issue :issue:`2105`) + +* The builtin ``bytearray`` type could not be used as base type of cdef classes. + (Github issue :issue:`2106`) + +Other changes +------------- + + +0.27.3 (2017-11-03) +=================== + +Bugs fixed +---------- + +* String forward references to extension types like ``@cython.locals(x="ExtType")`` + failed to find the named type. (Github issue :issue:`1962`) + +* NumPy slicing generated incorrect results when compiled with Pythran. + Original patch by Serge Guelton (Github issue :issue:`1946`). + +* Fix "undefined reference" linker error for generators on Windows in Py3.3-3.5. + (Github issue :issue:`1968`) + +* Adapt to recent C-API change of ``PyThreadState`` in CPython 3.7. + +* Fix signature of ``PyWeakref_GetObject()`` API declaration. + Patch by Jeroen Demeyer (Github issue :issue:`1975`). + + +0.27.2 (2017-10-22) +=================== + +Bugs fixed +---------- + +* Comprehensions could incorrectly be optimised away when they appeared in boolean + test contexts. (Github issue :issue:`1920`) + +* The special methods ``__eq__``, ``__lt__`` etc. in extension types did not type + their first argument as the type of the class but ``object``. (Github issue :issue:`1935`) + +* Crash on first lookup of "cline_in_traceback" option during exception handling. + (Github issue :issue:`1907`) + +* Some nested module level comprehensions failed to compile. + (Github issue :issue:`1906`) + +* Compiler crash on some complex type declarations in pure mode. + (Github issue :issue:`1908`) + +* ``std::unordered_map.erase()`` was declared with an incorrect ``void`` return + type in ``libcpp.unordered_map``. (Github issue :issue:`1484`) + +* Invalid use of C++ ``fallthrough`` attribute before C++11 and similar issue in clang. + (Github issue :issue:`1930`) + +* Compiler crash on misnamed properties. (Github issue :issue:`1905`) + + +0.27.1 (2017-10-01) +=================== + +Features added +-------------- + +* The Jupyter magic has a new debug option ``--verbose`` that shows details about + the distutils invocation. Patch by Boris Filippov (Github issue :issue:`1881`). + +Bugs fixed +---------- + +* Py3 list comprehensions in class bodies resulted in invalid C code. + (Github issue :issue:`1889`) + +* Modules built for later CPython 3.5.x versions failed to import in 3.5.0/3.5.1. + (Github issue :issue:`1880`) + +* Deallocating fused types functions and methods kept their GC tracking enabled, + which could potentially lead to recursive deallocation attempts. + +* Crash when compiling in C++ mode with old setuptools versions. + (Github issue :issue:`1879`) + +* C++ object arguments for the constructor of Cython implemented C++ are now + passed by reference and not by value to allow for non-copyable arguments, such + as ``unique_ptr``. + +* API-exported C++ classes with Python object members failed to compile. + (Github issue :issue:`1866`) + +* Some issues with the new relaxed exception value handling were resolved. + +* Python classes as annotation types could prevent compilation. + (Github issue :issue:`1887`) + +* Cython annotation types in Python files could lead to import failures + with a "cython undefined" error. Recognised types are now turned into strings. + +* Coverage analysis could fail to report on extension modules on some platforms. + +* Annotations could be parsed (and rejected) as types even with + ``annotation_typing=False``. + +Other changes +------------- + +* PEP 489 support has been disabled by default to counter incompatibilities with + import setups that try to reload or reinitialise modules. + + +0.27 (2017-09-23) +================= + +Features added +-------------- + +* Extension module initialisation follows + `PEP 489 `_ in CPython 3.5+, which + resolves several differences with regard to normal Python modules. This makes + the global names ``__file__`` and ``__path__`` correctly available to module + level code and improves the support for module-level relative imports. + (Github issues :issue:`1715`, :issue:`1753`, :issue:`1035`) + +* Asynchronous generators (`PEP 525 `_) + and asynchronous comprehensions (`PEP 530 `_) + have been implemented. Note that async generators require finalisation support + in order to allow for asynchronous operations during cleanup, which is only + available in CPython 3.6+. All other functionality has been backported as usual. + +* Variable annotations are now parsed according to + `PEP 526 `_. Cython types (e.g. + ``cython.int``) are evaluated as C type declarations and everything else as Python + types. This can be disabled with the directive ``annotation_typing=False``. + Note that most complex PEP-484 style annotations are currently ignored. This will + change in future releases. (Github issue :issue:`1850`) + +* Extension types (also in pure Python mode) can implement the normal special methods + ``__eq__``, ``__lt__`` etc. for comparisons instead of the low-level ``__richcmp__`` + method. (Github issue :issue:`690`) + +* New decorator ``@cython.exceptval(x=None, check=False)`` that makes the signature + declarations ``except x``, ``except? x`` and ``except *`` available to pure Python + code. Original patch by Antonio Cuni. (Github issue :issue:`1653`) + +* Signature annotations are now included in the signature docstring generated by + the ``embedsignature`` directive. Patch by Lisandro Dalcin (Github issue :issue:`1781`). + +* The gdb support for Python code (``libpython.py``) was updated to the latest + version in CPython 3.7 (git rev 5fe59f8). + +* The compiler tries to find a usable exception return value for cdef functions + with ``except *`` if the returned type allows it. Note that this feature is subject + to safety limitations, so it is still better to provide an explicit declaration. + +* C functions can be assigned to function pointers with a compatible exception + declaration, not only with exact matches. A side-effect is that certain compatible + signature overrides are now allowed and some more mismatches of exception signatures + are now detected and rejected as errors that were not detected before. + +* The IPython/Jupyter magic integration has a new option ``%%cython --pgo`` for profile + guided optimisation. It compiles the cell with PGO settings for the C compiler, + executes it to generate a runtime profile, and then compiles it again using that + profile for C compiler optimisation. Currently only tested with gcc. + +* ``len(memoryview)`` can be used in nogil sections to get the size of the + first dimension of a memory view (``shape[0]``). (Github issue :issue:`1733`) + +* C++ classes can now contain (properly refcounted) Python objects. + +* NumPy dtype subarrays are now accessible through the C-API. + Patch by Gerald Dalley (Github issue :issue:`245`). + +* Resolves several issues with PyPy and uses faster async slots in PyPy3. + Patch by Ronan Lamy (Github issues :issue:`1871`, :issue:`1878`). + +Bugs fixed +---------- + +* Extension types that were cimported from other Cython modules could disagree + about the order of fused cdef methods in their call table. This could lead + to wrong methods being called and potentially also crashes. The fix required + changes to the ordering of fused methods in the call table, which may break + existing compiled modules that call fused cdef methods across module boundaries, + if these methods were implemented in a different order than they were declared + in the corresponding .pxd file. (Github issue :issue:`1873`) + +* The exception state handling in generators and coroutines could lead to + exceptions in the caller being lost if an exception was raised and handled + inside of the coroutine when yielding. (Github issue :issue:`1731`) + +* Loops over ``range(enum)`` were not converted into C for-loops. Note that it + is still recommended to use an explicit cast to a C integer type in this case. + +* Error positions of names (e.g. variables) were incorrectly reported after the + name and not at the beginning of the name. + +* Compile time ``DEF`` assignments were evaluated even when they occur inside of + falsy ``IF`` blocks. (Github issue :issue:`1796`) + +* Disabling the line tracing from a trace function could fail. + Original patch by Dmitry Trofimov. (Github issue :issue:`1769`) + +* Several issues with the Pythran integration were resolved. + +* abs(signed int) now returns a signed rather than unsigned int. + (Github issue :issue:`1837`) + +* Reading ``frame.f_locals`` of a Cython function (e.g. from a debugger or profiler + could modify the module globals. (Github issue :issue:`1836`) + +* Buffer type mismatches in the NumPy buffer support could leak a reference to the + buffer owner. + +* Using the "is_f_contig" and "is_c_contig" memoryview methods together could leave + one of them undeclared. (Github issue :issue:`1872`) + +* Compilation failed if the for-in-range loop target was not a variable but a more + complex expression, e.g. an item assignment. (Github issue :issue:`1831`) + +* Compile time evaluations of (partially) constant f-strings could show incorrect + results. + +* Escape sequences in raw f-strings (``fr'...'``) were resolved instead of passing + them through as expected. + +* Some ref-counting issues in buffer error handling have been resolved. + +Other changes +------------- + +* Type declarations in signature annotations are now parsed according to + `PEP 484 `_ + typing. Only Cython types (e.g. ``cython.int``) and Python builtin types are + currently considered as type declarations. Everything else is ignored, but this + will change in a future Cython release. + (Github issue :issue:`1672`) + +* The directive ``annotation_typing`` is now ``True`` by default, which enables + parsing type declarations from annotations. + +* This release no longer supports Python 3.2. + + +0.26.1 (2017-08-29) +=================== + +Features added +-------------- + +Bugs fixed +---------- + +* ``cython.view.array`` was missing ``.__len__()``. + +* Extension types with a ``.pxd`` override for their ``__releasebuffer__`` slot + (e.g. as provided by Cython for the Python ``array.array`` type) could leak + a reference to the buffer owner on release, thus not freeing the memory. + (Github issue :issue:`1638`) + +* Auto-decoding failed in 0.26 for strings inside of C++ containers. + (Github issue :issue:`1790`) + +* Compile error when inheriting from C++ container types. + (Github issue :issue:`1788`) + +* Invalid C code in generators (declaration after code). + (Github issue :issue:`1801`) + +* Arithmetic operations on ``const`` integer variables could generate invalid code. + (Github issue :issue:`1798`) + +* Local variables with names of special Python methods failed to compile inside of + closures. (Github issue :issue:`1797`) + +* Problem with indirect Emacs buffers in cython-mode. + Patch by Martin Albrecht (Github issue :issue:`1743`). + +* Extension types named ``result`` or ``PickleError`` generated invalid unpickling code. + Patch by Jason Madden (Github issue :issue:`1786`). + +* Bazel integration failed to compile ``.py`` files. + Patch by Guro Bokum (Github issue :issue:`1784`). + +* Some include directories and dependencies were referenced with their absolute paths + in the generated files despite lying within the project directory. + +* Failure to compile in Py3.7 due to a modified signature of ``_PyCFunctionFast()`` + + +0.26 (2017-07-19) +================= + +Features added +-------------- + +* Pythran can be used as a backend for evaluating NumPy array expressions. + Patch by Adrien Guinet (Github issue :issue:`1607`). + +* cdef classes now support pickling by default when possible. + This can be disabled with the ``auto_pickle`` directive. + +* Speed up comparisons of strings if their hash value is available. + Patch by Claudio Freire (Github issue :issue:`1571`). + +* Support pyximport from zip files. + Patch by Sergei Lebedev (Github issue :issue:`1485`). + +* IPython magic now respects the ``__all__`` variable and ignores + names with leading-underscore (like ``import *`` does). + Patch by Syrtis Major (Github issue :issue:`1625`). + +* ``abs()`` is optimised for C complex numbers. + Patch by David Woods (Github issue :issue:`1648`). + +* The display of C lines in Cython tracebacks can now be enabled at runtime + via ``import cython_runtime; cython_runtime.cline_in_traceback=True``. + The default has been changed to False. + +* The overhead of calling fused types generic functions was reduced. + +* "cdef extern" include files are now also searched relative to the current file. + Patch by Jeroen Demeyer (Github issue :issue:`1654`). + +* Optional optimization for re-acquiring the GIL, controlled by the + `fast_gil` directive. + +Bugs fixed +---------- + +* Item lookup/assignment with a unicode character as index that is typed + (explicitly or implicitly) as ``Py_UCS4`` or ``Py_UNICODE`` used the + integer value instead of the Unicode string value. Code that relied on + the previous behaviour now triggers a warning that can be disabled by + applying an explicit cast. (Github issue :issue:`1602`) + +* f-string processing was adapted to changes in PEP 498 and CPython 3.6. + +* Invalid C code when decoding from UTF-16(LE/BE) byte strings. + (Github issue :issue:`1696`) + +* Unicode escapes in 'ur' raw-unicode strings were not resolved in Py2 code. + Original patch by Aaron Gallagher (Github issue :issue:`1594`). + +* File paths of code objects are now relative. + Original patch by Jelmer Vernooij (Github issue :issue:`1565`). + +* Decorators of cdef class methods could be executed twice. + Patch by Jeroen Demeyer (Github issue :issue:`1724`). + +* Dict iteration using the Py2 ``iter*`` methods failed in PyPy3. + Patch by Armin Rigo (Github issue :issue:`1631`). + +* Several warnings in the generated code are now suppressed. + +Other changes +------------- + +* The ``unraisable_tracebacks`` option now defaults to ``True``. + +* Coercion of C++ containers to Python is no longer automatic on attribute + access (Github issue :issue:`1521`). + +* Access to Python attributes of cimported modules without the corresponding + import is now a compile-time (rather than runtime) error. + +* Do not use special dll linkage for "cdef public" functions. + Patch by Jeroen Demeyer (Github issue :issue:`1687`). + +* cdef/cpdef methods must match their declarations. See Github issue :issue:`1732`. + This is now a warning and will be an error in future releases. + + +0.25.2 (2016-12-08) +=================== + +Bugs fixed +---------- + +* Fixes several issues with C++ template deduction. + +* Fixes a issue with bound method type inference (Github issue :issue:`551`). + +* Fixes a bug with cascaded tuple assignment (Github issue :issue:`1523`). + +* Fixed or silenced many Clang warnings. + +* Fixes bug with powers of pure real complex numbers (Github issue :issue:`1538`). + + +0.25.1 (2016-10-26) +=================== + +Bugs fixed +---------- + +* Fixes a bug with ``isinstance(o, Exception)`` (Github issue :issue:`1496`). + +* Fixes bug with ``cython.view.array`` missing utility code in some cases + (Github issue :issue:`1502`). + +Other changes +------------- + +* The distutils extension ``Cython.Distutils.build_ext`` has been reverted, + temporarily, to be ``old_build_ext`` to give projects time to migrate. + The new build_ext is available as ``new_build_ext``. + + +0.25 (2016-10-25) ================= Features added @@ -19,7 +2637,8 @@ * The new METH_FASTCALL calling convention for PyCFunctions is supported in CPython 3.6. See https://bugs.python.org/issue27810 -* Initial support for using Cython modules in Pyston. Patch by Daetalus. +* Initial support for using Cython modules in Pyston. + Patch by Boxiang Sun. * Dynamic Python attributes are allowed on cdef classes if an attribute ``cdef dict __dict__`` is declared in the class. Patch by empyrical. @@ -34,7 +2653,7 @@ Patch by Claudio Freire. * Buffer variables are no longer excluded from ``locals()``. - Patch by da-woods. + Patch by David Woods. * Building f-strings is faster, especially when formatting C integers. @@ -53,8 +2672,8 @@ * Support for bazel using a the pyx_library rule in //Tools:rules.bzl. -Bugs fixed ----------- +Significant Bugs fixed +---------------------- * Division of complex numbers avoids overflow by using Smith's method. @@ -72,6 +2691,9 @@ to use cythonize which properly handles dependencies. The old extension can still be found in ``Cython.Distutils.old_build_ext`` and is now deprecated. +* ``directive_defaults`` is no longer available in ``Cython.Compiler.Options``, + use ``get_directive_defaults()`` instead. + 0.24.1 (2016-07-15) =================== @@ -82,7 +2704,7 @@ * IPython cell magic was lacking a good way to enable Python 3 code semantics. It can now be used as "%%cython -3". -* Follow a recent change in `PEP 492 `_ +* Follow a recent change in `PEP 492 `_ and CPython 3.5.2 that now requires the ``__aiter__()`` method of asynchronous iterators to be a simple ``def`` method instead of an ``async def`` method. @@ -112,12 +2734,12 @@ Features added -------------- -* PEP 498: Literal String Formatting (f-strings). +* `PEP 498 `_: + Literal String Formatting (f-strings). Original patch by Jelle Zijlstra. - https://www.python.org/dev/peps/pep-0498/ -* PEP 515: Underscores as visual separators in number literals. - https://www.python.org/dev/peps/pep-0515/ +* `PEP 515 `_: + Underscores as visual separators in number literals. * Parser was adapted to some minor syntax changes in Py3.6, e.g. https://bugs.python.org/issue9232 @@ -179,7 +2801,7 @@ * Compile errors and warnings in integer type conversion code. This fixes ticket 877. Patches by Christian Neukirchen, Nikolaus Rath, Ian Henriksen. -* Reference leak when "*args" argument was reassigned in closures. +* Reference leak when ``*args`` argument was reassigned in closures. * Truth-testing Unicode strings could waste time and memory in Py3.3+. @@ -300,11 +2922,11 @@ Features added -------------- -* PEP 492 (async/await) was implemented. - See https://www.python.org/dev/peps/pep-0492/ +* `PEP 492 `_ + (async/await) was implemented. -* PEP 448 (Additional Unpacking Generalizations) was implemented. - See https://www.python.org/dev/peps/pep-0448/ +* `PEP 448 `_ + (Additional Unpacking Generalizations) was implemented. * Support for coverage.py 4.0+ can be enabled by adding the plugin "Cython.Coverage" to the ".coveragerc" config file. @@ -495,9 +3117,9 @@ * Anonymous C tuple types can be declared as (ctype1, ctype2, ...). -* PEP 479: turn accidental StopIteration exceptions that exit generators +* `PEP 479 `_: + turn accidental StopIteration exceptions that exit generators into a RuntimeError, activated with future import "generator_stop". - See https://www.python.org/dev/peps/pep-0479/ * Looping over ``reversed(range())`` is optimised in the same way as ``range()``. Patch by Favian Contreras. @@ -635,7 +3257,7 @@ * Generators have new properties ``__name__`` and ``__qualname__`` that provide the plain/qualified name of the generator function - (following CPython 3.5). See http://bugs.python.org/issue21205 + (following CPython 3.5). See https://bugs.python.org/issue21205 * The ``inline`` function modifier is available as a decorator ``@cython.inline`` in pure mode. @@ -681,7 +3303,7 @@ evaluation and generally improves the code flow in the generated C code. * The Python expression "2 ** N" is optimised into bit shifting. - See http://bugs.python.org/issue21420 + See https://bugs.python.org/issue21420 * Cascaded assignments (a = b = ...) try to minimise the number of type coercions. @@ -1470,11 +4092,11 @@ * Extension type inheritance from builtin types, such as "cdef class MyUnicode(unicode)", now works without further external type redeclarations (which are also strongly discouraged now and continue to issue a warning). -* GDB support. http://docs.cython.org/src/userguide/debugging.html +* GDB support. https://docs.cython.org/src/userguide/debugging.html -* A new build system with support for inline distutils directives, correct dependency tracking, and parallel compilation. http://wiki.cython.org/enhancements/distutils_preprocessing +* A new build system with support for inline distutils directives, correct dependency tracking, and parallel compilation. https://github.com/cython/cython/wiki/enhancements-distutils_preprocessing -* Support for dynamic compilation at runtime via the new cython.inline function and cython.compile decorator. http://wiki.cython.org/enhancements/inline +* Support for dynamic compilation at runtime via the new cython.inline function and cython.compile decorator. https://github.com/cython/cython/wiki/enhancements-inline * "nogil" blocks are supported when compiling pure Python code by writing "with cython.nogil". @@ -1650,7 +4272,7 @@ cdef np.ndarray[np.complex128_t, ndim=2] arr = \ np.zeros(10, np.complex128) -* Cython can now generate a main()-method for embedding of the Python interpreter into an executable (see #289) [Robert Bradshaw] +* Cython can now generate a main()-method for embedding of the Python interpreter into an executable (see :issue:`289`) [Robert Bradshaw] * @wraparound directive (another way to disable arr[idx] for negative idx) [Dag Sverre Seljebotn] diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/BuildExecutable.py cython-0.20.1+1~202203241016-9537/Cython/Build/BuildExecutable.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/BuildExecutable.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/BuildExecutable.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,10 @@ """ -Compile a Python script into an executable that embeds CPython and run it. +Compile a Python script into an executable that embeds CPython. Requires CPython to be built as a shared library ('libpythonX.Y'). Basic usage: - python cythonrun somefile.py [ARGS] + python -m Cython.Build.BuildExecutable [ARGS] somefile.py """ from __future__ import absolute_import @@ -28,7 +28,7 @@ # no shared library PYLIB_DYN = '' else: - PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ + PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ CC = get_config_var('CC', os.environ.get('CC', '')) CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') @@ -38,12 +38,14 @@ SYSLIBS = get_config_var('SYSLIBS') EXE_EXT = sysconfig.get_config_var('EXE') + def _debug(msg, *args): if DEBUG: if args: msg = msg % args sys.stderr.write(msg + '\n') + def dump_config(): _debug('INCDIR: %s', INCDIR) _debug('LIBDIR1: %s', LIBDIR1) @@ -58,6 +60,26 @@ _debug('SYSLIBS: %s', SYSLIBS) _debug('EXE_EXT: %s', EXE_EXT) + +def _parse_args(args): + cy_args = [] + last_arg = None + for i, arg in enumerate(args): + if arg.startswith('-'): + cy_args.append(arg) + elif last_arg in ('-X', '--directive'): + cy_args.append(arg) + else: + input_file = arg + args = args[i+1:] + break + last_arg = arg + else: + raise ValueError('no input file provided') + + return input_file, cy_args, args + + def runcmd(cmd, shell=True): if shell: cmd = ' '.join(cmd) @@ -65,24 +87,23 @@ else: _debug(' '.join(cmd)) - try: - import subprocess - except ImportError: # Python 2.3 ... - returncode = os.system(cmd) - else: - returncode = subprocess.call(cmd, shell=shell) - + import subprocess + returncode = subprocess.call(cmd, shell=shell) + if returncode: sys.exit(returncode) + def clink(basename): runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2] + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)] + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split()) + def ccompile(basename): runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split()) + def cycompile(input_file, options=()): from ..Compiler import Version, CmdLine, Main options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file]) @@ -91,9 +112,11 @@ if result.num_errors > 0: sys.exit(1) + def exec_file(program_name, args=()): runcmd([os.path.abspath(program_name)] + list(args), shell=False) + def build(input_file, compiler_args=(), force=False): """ Build an executable program from a Cython module. @@ -105,7 +128,7 @@ if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): raise ValueError("Input and output file names are the same, refusing to overwrite") if (not force and os.path.exists(exe_file) and os.path.exists(input_file) - and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): + and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): _debug("File is up to date, not regenerating %s", exe_file) return exe_file cycompile(input_file, compiler_args) @@ -113,30 +136,22 @@ clink(basename) return exe_file + def build_and_run(args): """ - Build an executable program from a Cython module and runs it. + Build an executable program from a Cython module and run it. - Arguments after the module name will be passed verbatimely to the - program. + Arguments after the module name will be passed verbatimly to the program. """ - cy_args = [] - last_arg = None - for i, arg in enumerate(args): - if arg.startswith('-'): - cy_args.append(arg) - elif last_arg in ('-X', '--directive'): - cy_args.append(arg) - else: - input_file = arg - args = args[i+1:] - break - last_arg = arg - else: - raise ValueError('no input file provided') + program_name, args = _build(args) + exec_file(program_name, args) + +def _build(args): + input_file, cy_args, args = _parse_args(args) program_name = build(input_file, cy_args) - exec_file(program_name, args) + return program_name, args + if __name__ == '__main__': - build_and_run(sys.argv[1:]) + _build(sys.argv[1:]) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Cythonize.py cython-0.20.1+1~202203241016-9537/Cython/Build/Cythonize.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Cythonize.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Cythonize.py 2022-03-24 10:16:46.000000000 +0000 @@ -21,40 +21,26 @@ class _FakePool(object): def map_async(self, func, args): - from itertools import imap + try: + from itertools import imap + except ImportError: + imap=map for _ in imap(func, args): pass - def close(self): pass - def terminate(self): pass - def join(self): pass - - -def parse_directives(option, name, value, parser): - dest = option.dest - old_directives = dict(getattr(parser.values, dest, - Options.get_directive_defaults())) - directives = Options.parse_directive_list( - value, relaxed_bool=True, current_settings=old_directives) - setattr(parser.values, dest, directives) - - -def parse_options(option, name, value, parser): - dest = option.dest - options = dict(getattr(parser.values, dest, {})) - for opt in value.split(','): - if '=' in opt: - n, v = opt.split('=', 1) - v = v.lower() not in ('false', 'f', '0', 'no') - else: - n, v = opt, True - options[n] = v - setattr(parser.values, dest, options) + def close(self): + pass + + def terminate(self): + pass + + def join(self): + pass def find_package_base(path): base_dir, package_path = os.path.split(path) - while os.path.isfile(os.path.join(base_dir, '__init__.py')): + while is_package_dir(base_dir): base_dir, parent = os.path.split(base_dir) package_path = '%s/%s' % (parent, package_path) return base_dir, package_path @@ -85,8 +71,10 @@ exclude_failures=options.keep_going, exclude=options.excludes, compiler_directives=options.directives, + compile_time_env=options.compile_time_env, force=options.force, quiet=options.quiet, + depfile=options.depfile, **options.options) if ext_modules and options.build: @@ -132,57 +120,90 @@ shutil.rmtree(temp_dir) -def parse_args(args): - from optparse import OptionParser - parser = OptionParser(usage='%prog [options] [sources and packages]+') - - parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', dest='directives', - type=str, action='callback', callback=parse_directives, default={}, +def create_args_parser(): + from argparse import ArgumentParser + from ..Compiler.CmdLine import ParseDirectivesAction, ParseOptionsAction, ParseCompileTimeEnvAction + + parser = ArgumentParser() + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='directives', default={}, type=str, + action=ParseDirectivesAction, help='set a compiler directive') - parser.add_option('-s', '--option', metavar='NAME=VALUE', dest='options', - type=str, action='callback', callback=parse_options, default={}, + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', default={}, type=str, + action=ParseCompileTimeEnvAction, + help='set a compile time environment variable') + parser.add_argument('-s', '--option', metavar='NAME=VALUE', + dest='options', default={}, type=str, + action=ParseOptionsAction, help='set a cythonize option') - parser.add_option('-3', dest='python3_mode', action='store_true', + parser.add_argument('-2', dest='language_level', action='store_const', const=2, default=None, + help='use Python 2 syntax mode by default') + parser.add_argument('-3', dest='language_level', action='store_const', const=3, help='use Python 3 syntax mode by default') - parser.add_option('-a', '--annotate', dest='annotate', action='store_true', - help='generate annotated HTML page for source files') - - parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', + help='use Python 3 syntax mode by default') + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument('-x', '--exclude', metavar='PATTERN', dest='excludes', action='append', default=[], help='exclude certain file patterns from the compilation') - parser.add_option('-b', '--build', dest='build', action='store_true', + parser.add_argument('-b', '--build', dest='build', action='store_true', default=None, help='build extension modules using distutils') - parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', + parser.add_argument('-i', '--inplace', dest='build_inplace', action='store_true', default=None, help='build extension modules in place using distutils (implies -b)') - parser.add_option('-j', '--parallel', dest='parallel', metavar='N', + parser.add_argument('-j', '--parallel', dest='parallel', metavar='N', type=int, default=parallel_compiles, help=('run builds in N parallel jobs (default: %d)' % parallel_compiles or 1)) - parser.add_option('-f', '--force', dest='force', action='store_true', + parser.add_argument('-f', '--force', dest='force', action='store_true', default=None, help='force recompilation') - parser.add_option('-q', '--quiet', dest='quiet', action='store_true', + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=None, help='be less verbose during compilation') - parser.add_option('--lenient', dest='lenient', action='store_true', + parser.add_argument('--lenient', dest='lenient', action='store_true', default=None, help='increase Python compatibility by ignoring some compile time errors') - parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', + parser.add_argument('-k', '--keep-going', dest='keep_going', action='store_true', default=None, help='compile as much as possible, ignore compilation failures') + parser.add_argument('--no-docstrings', dest='no_docstrings', action='store_true', default=None, + help='strip docstrings') + parser.add_argument('-M', '--depfile', action='store_true', help='produce depfiles for the sources') + parser.add_argument('sources', nargs='*') + return parser + + +def parse_args_raw(parser, args): + options, unknown = parser.parse_known_args(args) + sources = options.sources + # if positional arguments were interspersed + # some of them are in unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option "+option) + else: + sources.append(option) + del options.sources + return (options, sources) + + +def parse_args(args): + parser = create_args_parser() + options, args = parse_args_raw(parser, args) - options, args = parser.parse_args(args) if not args: parser.error("no source files provided") if options.build_inplace: options.build = True if multiprocessing is None: options.parallel = 0 - if options.python3_mode: - options.options['language_level'] = 3 - return options, args - - -def main(args=None): - options, paths = parse_args(args) + if options.language_level: + assert options.language_level in (2, 3, '3str') + options.options['language_level'] = options.language_level if options.lenient: # increase Python compatibility by ignoring compile time errors @@ -190,7 +211,16 @@ Options.error_on_uninitialized = False if options.annotate: - Options.annotate = True + Options.annotate = options.annotate + + if options.no_docstrings: + Options.docstrings = False + + return options, args + + +def main(args=None): + options, paths = parse_args(args) for path in paths: cython_compile(path, options) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Dependencies.py cython-0.20.1+1~202203241016-9537/Cython/Build/Dependencies.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Dependencies.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Dependencies.py 2022-03-24 10:16:46.000000000 +0000 @@ -4,8 +4,23 @@ from .. import __version__ import collections -import re, os, sys, time +import contextlib +import hashlib +import os +import shutil +import subprocess +import re, sys, time from glob import iglob +from io import open as io_open +from os.path import relpath as _relpath +from distutils.extension import Extension +from distutils.util import strtobool +import zipfile + +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable try: import gzip @@ -14,41 +29,24 @@ except ImportError: gzip_open = open gzip_ext = '' -import shutil -import subprocess - -try: - import hashlib -except ImportError: - import md5 as hashlib try: - from io import open as io_open + import zlib + zipfile_compression_mode = zipfile.ZIP_DEFLATED except ImportError: - from codecs import open as io_open + zipfile_compression_mode = zipfile.ZIP_STORED try: - from os.path import relpath as _relpath -except ImportError: - # Py<2.6 - def _relpath(path, start=os.path.curdir): - if not path: - raise ValueError("no path specified") - start_list = os.path.abspath(start).split(os.path.sep) - path_list = os.path.abspath(path).split(os.path.sep) - i = len(os.path.commonprefix([start_list, path_list])) - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.path.curdir - return os.path.join(*rel_list) - - -from distutils.extension import Extension + import pythran +except: + pythran = None from .. import Utils from ..Utils import (cached_function, cached_method, path_exists, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir) -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler import Errors +from ..Compiler.Main import Context +from ..Compiler.Options import CompilationOptions, default_options join_path = cached_function(os.path.join) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) @@ -69,6 +67,15 @@ basestring = str +def _make_relative(file_paths, base=None): + if not base: + base = os.getcwd() + if base[-1] != os.path.sep: + base += os.path.sep + return [_relpath(path, base) if path.startswith(base) else path + for path in file_paths] + + def extended_iglob(pattern): if '{' in pattern: m = re.match('(.*){([^}]+)}(.*)', pattern) @@ -110,20 +117,42 @@ @cached_function def file_hash(filename): - path = os.path.normpath(filename.encode("UTF-8")) - m = hashlib.md5(str(len(path)) + ":") - m.update(path) - f = open(filename, 'rb') - try: + path = os.path.normpath(filename) + prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") + m = hashlib.sha1(prefix) + with open(path, 'rb') as f: data = f.read(65000) while data: m.update(data) data = f.read(65000) - finally: - f.close() return m.hexdigest() +def update_pythran_extension(ext): + if pythran is None: + raise RuntimeError("You first need to install Pythran to use the np_pythran directive.") + try: + pythran_ext = pythran.config.make_extension(python=True) + except TypeError: # older pythran version only + pythran_ext = pythran.config.make_extension() + + ext.include_dirs.extend(pythran_ext['include_dirs']) + ext.extra_compile_args.extend(pythran_ext['extra_compile_args']) + ext.extra_link_args.extend(pythran_ext['extra_link_args']) + ext.define_macros.extend(pythran_ext['define_macros']) + ext.undef_macros.extend(pythran_ext['undef_macros']) + ext.library_dirs.extend(pythran_ext['library_dirs']) + ext.libraries.extend(pythran_ext['libraries']) + ext.language = 'c++' + + # These options are not compatible with the way normal Cython extensions work + for bad_option in ["-fwhole-program", "-fvisibility=hidden"]: + try: + ext.extra_compile_args.remove(bad_option) + except ValueError: + pass + + def parse_list(s): """ >>> parse_list("") @@ -156,6 +185,7 @@ transitive_str = object() transitive_list = object() +bool_or = object() distutils_settings = { 'name': str, @@ -172,6 +202,7 @@ 'export_symbols': list, 'depends': transitive_list, 'language': transitive_str, + 'np_pythran': bool_or } @@ -203,19 +234,23 @@ if line[0] != '#': break line = line[1:].lstrip() - if line[:10] == 'distutils:': - key, _, value = [s.strip() for s in line[10:].partition('=')] - type = distutils_settings[key] + kind = next((k for k in ("distutils:","cython:") if line.startswith(k)), None) + if kind is not None: + key, _, value = [s.strip() for s in line[len(kind):].partition('=')] + type = distutils_settings.get(key, None) + if line.startswith("cython:") and type is None: continue if type in (list, transitive_list): value = parse_list(value) if key == 'define_macros': value = [tuple(macro.split('=', 1)) if '=' in macro else (macro, None) for macro in value] + if type is bool_or: + value = strtobool(value) self.values[key] = value elif exn is not None: for key in distutils_settings: - if key in ('name', 'sources'): + if key in ('name', 'sources','np_pythran'): continue value = getattr(exn, key, None) if value: @@ -237,6 +272,8 @@ all.append(v) value = all self.values[key] = value + elif type is bool_or: + self.values[key] = self.values.get(key, False) | value return self def subs(self, aliases): @@ -286,7 +323,8 @@ in_quote = False hash_mark = single_q = double_q = -1 code_len = len(code) - quote_type = quote_len = None + quote_type = None + quote_len = -1 while True: if hash_mark < q: @@ -366,6 +404,10 @@ r"(?:^\s*cimport +([0-9a-zA-Z_.]+(?: *, *[0-9a-zA-Z_.]+)*))|" r"(?:^\s*cdef +extern +from +['\"]([^'\"]+)['\"])|" r"(?:^\s*include +['\"]([^'\"]+)['\"])", re.M) +dependency_after_from_regex = re.compile( + r"(?:^\s+\(([0-9a-zA-Z_., ]*)\)[#\n])|" + r"(?:^\s+([0-9a-zA-Z_., ]*)[#\n])", + re.M) def normalize_existing(base_path, rel_paths): @@ -374,14 +416,30 @@ @cached_function def normalize_existing0(base_dir, rel_paths): + """ + Given some base directory ``base_dir`` and a list of path names + ``rel_paths``, normalize each relative path name ``rel`` by + replacing it by ``os.path.join(base, rel)`` if that file exists. + + Return a couple ``(normalized, needed_base)`` where ``normalized`` + if the list of normalized file names and ``needed_base`` is + ``base_dir`` if we actually needed ``base_dir``. If no paths were + changed (for example, if all paths were already absolute), then + ``needed_base`` is ``None``. + """ normalized = [] + needed_base = None for rel in rel_paths: + if os.path.isabs(rel): + normalized.append(rel) + continue path = join_path(base_dir, rel) if path_exists(path): normalized.append(os.path.normpath(path)) + needed_base = base_dir else: normalized.append(rel) - return normalized + return (normalized, needed_base) def resolve_depends(depends, include_dirs): @@ -425,11 +483,8 @@ # Actual parsing is way too slow, so we use regular expressions. # The only catch is that we must strip comments and string # literals ahead of time. - fh = Utils.open_source_file(source_filename, error_handling='ignore') - try: + with Utils.open_source_file(source_filename, error_handling='ignore') as fh: source = fh.read() - finally: - fh.close() distutils_info = DistutilsInfo(source) source, literals = strip_string_literals(source) source = source.replace('\\\n', ' ').replace('\t', ' ') @@ -442,6 +497,13 @@ cimport_from, cimport_list, extern, include = m.groups() if cimport_from: cimports.append(cimport_from) + m_after_from = dependency_after_from_regex.search(source, pos=m.end()) + if m_after_from: + multiline, one_line = m_after_from.groups() + subimports = multiline or one_line + cimports.extend("{0}.{1}".format(cimport_from, s.strip()) + for s in subimports.split(',')) + elif cimport_list: cimports.extend(x.strip() for x in cimport_list.split(",")) elif extern: @@ -471,31 +533,36 @@ for include in self.parse_dependencies(filename)[1]: include_path = join_path(os.path.dirname(filename), include) if not path_exists(include_path): - include_path = self.context.find_include_file(include, None) + include_path = self.context.find_include_file(include, source_file_path=filename) if include_path: if '.' + os.path.sep in include_path: include_path = os.path.normpath(include_path) all.add(include_path) all.update(self.included_files(include_path)) elif not self.quiet: - print("Unable to locate '%s' referenced from '%s'" % (filename, include)) + print(u"Unable to locate '%s' referenced from '%s'" % (filename, include)) return all @cached_method - def cimports_and_externs(self, filename): + def cimports_externs_incdirs(self, filename): # This is really ugly. Nested cimports are resolved with respect to the # includer, but includes are resolved with respect to the includee. cimports, includes, externs = self.parse_dependencies(filename)[:3] cimports = set(cimports) externs = set(externs) + incdirs = set() for include in self.included_files(filename): - included_cimports, included_externs = self.cimports_and_externs(include) + included_cimports, included_externs, included_incdirs = self.cimports_externs_incdirs(include) cimports.update(included_cimports) externs.update(included_externs) - return tuple(cimports), normalize_existing(filename, externs) + incdirs.update(included_incdirs) + externs, incdir = normalize_existing(filename, externs) + if incdir: + incdirs.add(incdir) + return tuple(cimports), externs, incdirs def cimports(self, filename): - return self.cimports_and_externs(filename)[0] + return self.cimports_externs_incdirs(filename)[0] def package(self, filename): return package(filename) @@ -520,35 +587,36 @@ return None # FIXME: error? module_path.pop(0) relative = '.'.join(package_path + module_path) - pxd = self.context.find_pxd_file(relative, None) + pxd = self.context.find_pxd_file(relative, source_file_path=filename) if pxd: return pxd if is_relative: return None # FIXME: error? - return self.context.find_pxd_file(module, None) + return self.context.find_pxd_file(module, source_file_path=filename) @cached_method def cimported_files(self, filename): - if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): - pxd_list = [filename[:-4] + '.pxd'] + filename_root, filename_ext = os.path.splitext(filename) + if filename_ext in ('.pyx', '.py') and path_exists(filename_root + '.pxd'): + pxd_list = [filename_root + '.pxd'] else: pxd_list = [] + # Cimports generates all possible combinations package.module + # when imported as from package cimport module. for module in self.cimports(filename): if module[:7] == 'cython.' or module == 'cython': continue pxd_file = self.find_pxd(module, filename) if pxd_file is not None: pxd_list.append(pxd_file) - elif not self.quiet: - print("%s: cannot find cimported module '%s'" % (filename, module)) return tuple(pxd_list) @cached_method def immediate_dependencies(self, filename): - all = set([filename]) - all.update(self.cimported_files(filename)) - all.update(self.included_files(filename)) - return all + all_deps = {filename} + all_deps.update(self.cimported_files(filename)) + all_deps.update(self.included_files(filename)) + return all_deps def all_dependencies(self, filename): return self.transitive_merge(filename, self.immediate_dependencies, set.union) @@ -563,27 +631,56 @@ def newest_dependency(self, filename): return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)]) - def transitive_fingerprint(self, filename, extra=None): + def transitive_fingerprint(self, filename, module, compilation_options): + r""" + Return a fingerprint of a cython file that is about to be cythonized. + + Fingerprints are looked up in future compilations. If the fingerprint + is found, the cythonization can be skipped. The fingerprint must + incorporate everything that has an influence on the generated code. + """ try: - m = hashlib.md5(__version__) - m.update(file_hash(filename)) + m = hashlib.sha1(__version__.encode('UTF-8')) + m.update(file_hash(filename).encode('UTF-8')) for x in sorted(self.all_dependencies(filename)): if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): - m.update(file_hash(x)) - if extra is not None: - m.update(str(extra)) + m.update(file_hash(x).encode('UTF-8')) + # Include the module attributes that change the compilation result + # in the fingerprint. We do not iterate over module.__dict__ and + # include almost everything here as users might extend Extension + # with arbitrary (random) attributes that would lead to cache + # misses. + m.update(str(( + module.language, + getattr(module, 'py_limited_api', False), + getattr(module, 'np_pythran', False) + )).encode('UTF-8')) + + m.update(compilation_options.get_fingerprint().encode('UTF-8')) return m.hexdigest() except IOError: return None def distutils_info0(self, filename): info = self.parse_dependencies(filename)[3] - externs = self.cimports_and_externs(filename)[1] + kwds = info.values + cimports, externs, incdirs = self.cimports_externs_incdirs(filename) + basedir = os.getcwd() + # Add dependencies on "cdef extern from ..." files if externs: - if 'depends' in info.values: - info.values['depends'] = list(set(info.values['depends']).union(externs)) + externs = _make_relative(externs, basedir) + if 'depends' in kwds: + kwds['depends'] = list(set(kwds['depends']).union(externs)) else: - info.values['depends'] = list(externs) + kwds['depends'] = list(externs) + # Add include_dirs to ensure that the C compiler will find the + # "cdef extern from ..." files + if incdirs: + include_dirs = list(kwds.get('include_dirs', [])) + for inc in _make_relative(incdirs, basedir): + if inc not in include_dirs: + include_dirs.append(inc) + kwds['include_dirs'] = include_dirs return info def distutils_info(self, filename, aliases=None, base=None): @@ -636,18 +733,33 @@ return _dep_tree +# If this changes, change also docs/src/reference/compilation.rst +# which mentions this function +def default_create_extension(template, kwds): + if 'depends' in kwds: + include_dirs = kwds.get('include_dirs', []) + ["."] + depends = resolve_depends(kwds['depends'], include_dirs) + kwds['depends'] = sorted(set(depends + template.depends)) + + t = template.__class__ + ext = t(**kwds) + metadata = dict(distutils=kwds, module_name=kwds['name']) + return (ext, metadata) + + # This may be useful for advanced users? def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=False, language=None, exclude_failures=False): if language is not None: - print('Please put "# distutils: language=%s" in your .pyx or .pxd file(s)' % language) + print('Warning: passing language={0!r} to cythonize() is deprecated. ' + 'Instead, put "# distutils: language={0}" in your .pyx or .pxd file(s)'.format(language)) if exclude is None: exclude = [] if patterns is None: return [], {} - elif isinstance(patterns, basestring) or not isinstance(patterns, collections.Iterable): + elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): patterns = [patterns] - explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) + explicit_modules = {m.name for m in patterns if isinstance(m, Extension)} seen = set() deps = create_dependency_tree(ctx, quiet=quiet) to_exclude = set() @@ -668,18 +780,28 @@ Extension_distutils = Extension class Extension_setuptools(Extension): pass + # if no create_extension() function is defined, use a simple + # default function. + create_extension = ctx.options.create_extension or default_create_extension + for pattern in patterns: + if not isinstance(pattern, (Extension_distutils, Extension_setuptools)): + pattern = encode_filename_in_py2(pattern) if isinstance(pattern, str): filepattern = pattern - template = None + template = Extension(pattern, []) # Fake Extension without sources name = '*' base = None - exn_type = Extension ext_language = language elif isinstance(pattern, (Extension_distutils, Extension_setuptools)): - for filepattern in pattern.sources: - if os.path.splitext(filepattern)[1] in ('.py', '.pyx'): - break + cython_sources = [s for s in pattern.sources + if os.path.splitext(s)[1] in ('.py', '.pyx')] + if cython_sources: + filepattern = cython_sources[0] + if len(cython_sources) > 1: + print(u"Warning: Multiple cython sources found for extension '%s': %s\n" + u"See https://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " + u"for sharing declarations among Cython files." % (pattern.name, cython_sources)) else: # ignore non-cython modules module_list.append(pattern) @@ -687,7 +809,6 @@ template = pattern name = template.name base = DistutilsInfo(exn=template) - exn_type = template.__class__ ext_language = None # do not override whatever the Extension says else: msg = str("pattern is not of type str nor subclass of Extension (%s)" @@ -699,16 +820,15 @@ for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern): if os.path.abspath(file) in to_exclude: continue - pkg = deps.package(file) module_name = deps.fully_qualified_name(file) if '*' in name: if module_name in explicit_modules: continue - elif name != module_name: - print("Warning: Extension name '%s' does not match fully qualified name '%s' of '%s'" % ( - name, module_name, file)) + elif name: module_name = name + Utils.raise_error_if_module_name_forbidden(module_name) + if module_name not in seen: try: kwds = deps.distutils_info(file, aliases, base).values @@ -721,83 +841,146 @@ if key not in kwds: kwds[key] = value - sources = [file] - if template is not None: - sources += [m for m in template.sources if m != filepattern] + kwds['name'] = module_name + + sources = [file] + [m for m in template.sources if m != filepattern] if 'sources' in kwds: # allow users to add .c files etc. for source in kwds['sources']: source = encode_filename_in_py2(source) if source not in sources: sources.append(source) - extra_sources = kwds['sources'] - del kwds['sources'] - else: - extra_sources = None - if 'depends' in kwds: - depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + ["."]) - if template is not None: - # Always include everything from the template. - depends = set(template.depends).union(depends) - # Sort depends to make the metadata dump in the - # Cython-generated C code predictable. - kwds['depends'] = sorted(depends) + kwds['sources'] = sources if ext_language and 'language' not in kwds: kwds['language'] = ext_language - module_list.append(exn_type( - name=module_name, - sources=sources, - **kwds)) - if extra_sources: - kwds['sources'] = extra_sources - module_metadata[module_name] = {'distutils': kwds, 'module_name': module_name} - m = module_list[-1] + np_pythran = kwds.pop('np_pythran', False) + + # Create the new extension + m, metadata = create_extension(template, kwds) + m.np_pythran = np_pythran or getattr(m, 'np_pythran', False) + if m.np_pythran: + update_pythran_extension(m) + module_list.append(m) + + # Store metadata (this will be written as JSON in the + # generated C file but otherwise has no purpose) + module_metadata[module_name] = metadata + + if file not in m.sources: + # Old setuptools unconditionally replaces .pyx with .c/.cpp + target_file = os.path.splitext(file)[0] + ('.cpp' if m.language == 'c++' else '.c') + try: + m.sources.remove(target_file) + except ValueError: + # never seen this in the wild, but probably better to warn about this unexpected case + print(u"Warning: Cython source file not found in sources list, adding %s" % file) + m.sources.insert(0, file) seen.add(name) return module_list, module_metadata # This is the user-exposed entry point. -def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, - exclude_failures=False, **options): +def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=None, language=None, + exclude_failures=False, show_all_warnings=False, **options): """ Compile a set of source modules into C/C++ files and return a list of distutils Extension objects for them. - As module list, pass either a glob pattern, a list of glob patterns or a list of - Extension objects. The latter allows you to configure the extensions separately - through the normal distutils options. - - When using glob patterns, you can exclude certain module names explicitly - by passing them into the 'exclude' option. - - To globally enable C++ mode, you can pass language='c++'. Otherwise, this - will be determined at a per-file level based on compiler directives. This - affects only modules found based on file names. Extension instances passed - into cythonize() will not be changed. - - For parallel compilation, set the 'nthreads' option to the number of - concurrent builds. - - For a broad 'try to compile' mode that ignores compilation failures and - simply excludes the failed extensions, pass 'exclude_failures=True'. Note - that this only really makes sense for compiling .py files which can also - be used without compilation. + :param module_list: As module list, pass either a glob pattern, a list of glob + patterns or a list of Extension objects. The latter + allows you to configure the extensions separately + through the normal distutils options. + You can also pass Extension objects that have + glob patterns as their sources. Then, cythonize + will resolve the pattern and create a + copy of the Extension for every matching file. + + :param exclude: When passing glob patterns as ``module_list``, you can exclude certain + module names explicitly by passing them into the ``exclude`` option. + + :param nthreads: The number of concurrent builds for parallel compilation + (requires the ``multiprocessing`` module). + + :param aliases: If you want to use compiler directives like ``# distutils: ...`` but + can only know at compile time (when running the ``setup.py``) which values + to use, you can use aliases and pass a dictionary mapping those aliases + to Python strings when calling :func:`cythonize`. As an example, say you + want to use the compiler + directive ``# distutils: include_dirs = ../static_libs/include/`` + but this path isn't always fixed and you want to find it when running + the ``setup.py``. You can then do ``# distutils: include_dirs = MY_HEADERS``, + find the value of ``MY_HEADERS`` in the ``setup.py``, put it in a python + variable called ``foo`` as a string, and then call + ``cythonize(..., aliases={'MY_HEADERS': foo})``. + + :param quiet: If True, Cython won't print error, warning, or status messages during the + compilation. + + :param force: Forces the recompilation of the Cython modules, even if the timestamps + don't indicate that a recompilation is necessary. + + :param language: To globally enable C++ mode, you can pass ``language='c++'``. Otherwise, this + will be determined at a per-file level based on compiler directives. This + affects only modules found based on file names. Extension instances passed + into :func:`cythonize` will not be changed. It is recommended to rather + use the compiler directive ``# distutils: language = c++`` than this option. + + :param exclude_failures: For a broad 'try to compile' mode that ignores compilation + failures and simply excludes the failed extensions, + pass ``exclude_failures=True``. Note that this only + really makes sense for compiling ``.py`` files which can also + be used without compilation. + + :param show_all_warnings: By default, not all Cython warnings are printed. + Set to true to show all warnings. + + :param annotate: If ``True``, will produce a HTML file for each of the ``.pyx`` or ``.py`` + files compiled. The HTML file gives an indication + of how much Python interaction there is in + each of the source code lines, compared to plain C code. + It also allows you to see the C/C++ code + generated for each line of Cython code. This report is invaluable when + optimizing a function for speed, + and for determining when to :ref:`release the GIL `: + in general, a ``nogil`` block may contain only "white" code. + See examples in :ref:`determining_where_to_add_types` or + :ref:`primes`. + + + :param annotate-fullc: If ``True`` will produce a colorized HTML version of + the source which includes entire generated C/C++-code. + + + :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this: + ``compiler_directives={'embedsignature': True}``. + See :ref:`compiler-directives`. - Additional compilation options can be passed as keyword arguments. + :param depfile: produce depfiles for the sources if True. """ if exclude is None: exclude = [] if 'include_path' not in options: options['include_path'] = ['.'] if 'common_utility_include_dir' in options: - if options.get('cache'): - raise NotImplementedError("common_utility_include_dir does not yet work with caching") safe_makedirs(options['common_utility_include_dir']) + + depfile = options.pop('depfile', None) + + if pythran is None: + pythran_options = None + else: + pythran_options = CompilationOptions(**options) + pythran_options.cplus = True + pythran_options.np_pythran = True + + if force is None: + force = os.environ.get("CYTHON_FORCE_REGEN") == "1" # allow global overrides for build systems + c_options = CompilationOptions(**options) cpp_options = CompilationOptions(**options); cpp_options.cplus = True - ctx = c_options.create_context() + ctx = Context.from_options(c_options) options = c_options module_list, module_metadata = create_extension_list( module_list, @@ -807,30 +990,47 @@ exclude_failures=exclude_failures, language=language, aliases=aliases) + + fix_windows_unicode_modules(module_list) + deps = create_dependency_tree(ctx, quiet=quiet) build_dir = getattr(options, 'build_dir', None) - modules_by_cfile = {} + def copy_to_build_dir(filepath, root=os.getcwd()): + filepath_abs = os.path.abspath(filepath) + if os.path.isabs(filepath): + filepath = filepath_abs + if filepath_abs.startswith(root): + # distutil extension depends are relative to cwd + mod_dir = join_path(build_dir, + os.path.dirname(_relpath(filepath, root))) + copy_once_if_newer(filepath_abs, mod_dir) + + modules_by_cfile = collections.defaultdict(list) to_compile = [] for m in module_list: if build_dir: - root = os.getcwd() # distutil extension depends are relative to cwd - def copy_to_build_dir(filepath, root=root): - filepath_abs = os.path.abspath(filepath) - if os.path.isabs(filepath): - filepath = filepath_abs - if filepath_abs.startswith(root): - mod_dir = join_path(build_dir, - os.path.dirname(_relpath(filepath, root))) - copy_once_if_newer(filepath_abs, mod_dir) for dep in m.depends: copy_to_build_dir(dep) + cy_sources = [ + source for source in m.sources + if os.path.splitext(source)[1] in ('.pyx', '.py')] + if len(cy_sources) == 1: + # normal "special" case: believe the Extension module name to allow user overrides + full_module_name = m.name + else: + # infer FQMN from source files + full_module_name = None + new_sources = [] for source in m.sources: base, ext = os.path.splitext(source) if ext in ('.pyx', '.py'): - if m.language == 'c++': + if m.np_pythran: + c_file = base + '.cpp' + options = pythran_options + elif m.language == 'c++': c_file = base + '.cpp' options = cpp_options else: @@ -839,11 +1039,34 @@ # setup for out of place build directory if enabled if build_dir: + if os.path.isabs(c_file): + c_file = os.path.splitdrive(c_file)[1] + c_file = c_file.split(os.sep, 1)[1] c_file = os.path.join(build_dir, c_file) dir = os.path.dirname(c_file) safe_makedirs_once(dir) - if os.path.exists(c_file): + # write out the depfile, if requested + if depfile: + dependencies = deps.all_dependencies(source) + src_base_dir, _ = os.path.split(source) + if not src_base_dir.endswith(os.sep): + src_base_dir += os.sep + # paths below the base_dir are relative, otherwise absolute + paths = [] + for fname in dependencies: + if fname.startswith(src_base_dir): + paths.append(os.path.relpath(fname, src_base_dir)) + else: + paths.append(os.path.abspath(fname)) + + depline = os.path.split(c_file)[1] + ": \\\n " + depline += " \\\n ".join(paths) + "\n" + with open(c_file+'.dep', 'w') as outfile: + outfile.write(depline) + + # Missing files and those generated by other Cython versions should always be recreated. + if Utils.file_generated_by_this_cython(c_file): c_timestamp = os.path.getmtime(c_file) else: c_timestamp = -1 @@ -857,23 +1080,24 @@ dep_timestamp, dep = deps.newest_dependency(source) priority = 2 - (dep in deps.immediate_dependencies(source)) if force or c_timestamp < dep_timestamp: - if not quiet: + if not quiet and not force: if source == dep: - print("Compiling %s because it changed." % source) + print(u"Compiling %s because it changed." % Utils.decode_filename(source)) else: - print("Compiling %s because it depends on %s." % (source, dep)) + print(u"Compiling %s because it depends on %s." % ( + Utils.decode_filename(source), + Utils.decode_filename(dep), + )) if not force and options.cache: - extra = m.language - fingerprint = deps.transitive_fingerprint(source, extra) + fingerprint = deps.transitive_fingerprint(source, m, options) else: fingerprint = None - to_compile.append((priority, source, c_file, fingerprint, quiet, - options, not exclude_failures, module_metadata.get(m.name))) + to_compile.append(( + priority, source, c_file, fingerprint, quiet, + options, not exclude_failures, module_metadata.get(m.name), + full_module_name, show_all_warnings)) new_sources.append(c_file) - if c_file not in modules_by_cfile: - modules_by_cfile[c_file] = [m] - else: - modules_by_cfile[c_file].append(m) + modules_by_cfile[c_file].append(m) else: new_sources.append(source) if build_dir: @@ -895,32 +1119,26 @@ if N <= 1: nthreads = 0 if nthreads: - # Requires multiprocessing (or Python >= 2.6) + import multiprocessing + pool = multiprocessing.Pool( + nthreads, initializer=_init_multiprocessing_helper) + # This is a bit more involved than it should be, because KeyboardInterrupts + # break the multiprocessing workers when using a normal pool.map(). + # See, for example: + # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt try: - import multiprocessing - pool = multiprocessing.Pool( - nthreads, initializer=_init_multiprocessing_helper) - except (ImportError, OSError): - print("multiprocessing required for parallel cythonization") - nthreads = 0 - else: - # This is a bit more involved than it should be, because KeyboardInterrupts - # break the multiprocessing workers when using a normal pool.map(). - # See, for example: - # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt - try: - result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) - pool.close() - while not result.ready(): - try: - result.get(99999) # seconds - except multiprocessing.TimeoutError: - pass - except KeyboardInterrupt: - pool.terminate() - raise - pool.join() - if not nthreads: + result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) + pool.close() + while not result.ready(): + try: + result.get(99999) # seconds + except multiprocessing.TimeoutError: + pass + except KeyboardInterrupt: + pool.terminate() + raise + pool.join() + else: for args in to_compile: cythonize_one(*args) @@ -940,7 +1158,7 @@ if failed_modules: for module in failed_modules: module_list.remove(module) - print("Failed compilations: %s" % ', '.join(sorted([ + print(u"Failed compilations: %s" % ', '.join(sorted([ module.name for module in failed_modules]))) if options.cache: @@ -951,6 +1169,41 @@ return module_list +def fix_windows_unicode_modules(module_list): + # Hack around a distutils 3.[5678] bug on Windows for unicode module names. + # https://bugs.python.org/issue39432 + if sys.platform != "win32": + return + if sys.version_info < (3, 5) or sys.version_info >= (3, 8, 2): + return + + def make_filtered_list(ignored_symbol, old_entries): + class FilteredExportSymbols(list): + # export_symbols for unicode filename cause link errors on Windows + # Cython doesn't need them (it already defines PyInit with the correct linkage) + # so use this class as a temporary fix to stop them from being generated + def __contains__(self, val): + # so distutils doesn't "helpfully" add PyInit_ + return val == ignored_symbol or list.__contains__(self, val) + + filtered_list = FilteredExportSymbols(old_entries) + if old_entries: + filtered_list.extend(name for name in old_entries if name != ignored_symbol) + return filtered_list + + for m in module_list: + # TODO: use m.name.isascii() in Py3.7+ + try: + m.name.encode("ascii") + continue + except UnicodeEncodeError: + pass + m.export_symbols = make_filtered_list( + "PyInit_" + m.name.rsplit(".", 1)[-1], + m.export_symbols, + ) + + if os.environ.get('XML_RESULTS'): compile_result_dir = os.environ['XML_RESULTS'] def record_results(func): @@ -989,45 +1242,51 @@ # TODO: Share context? Issue: pyx processing leaks into pxd module @record_results -def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True, embedded_metadata=None, progress=""): - from ..Compiler.Main import compile, default_options +def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, + raise_on_failure=True, embedded_metadata=None, + full_module_name=None, show_all_warnings=False, + progress=""): + from ..Compiler.Main import compile_single, default_options from ..Compiler.Errors import CompileError, PyrexError if fingerprint: if not os.path.exists(options.cache): - try: - os.mkdir(options.cache) - except: - if not os.path.exists(options.cache): - raise + safe_makedirs(options.cache) # Cython-generated c files are highly compressible. # (E.g. a compression ratio of about 10 for Sage). - fingerprint_file = join_path( - options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext)) - if os.path.exists(fingerprint_file): + fingerprint_file_base = join_path( + options.cache, "%s-%s" % (os.path.basename(c_file), fingerprint)) + gz_fingerprint_file = fingerprint_file_base + gzip_ext + zip_fingerprint_file = fingerprint_file_base + '.zip' + if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): if not quiet: - print("%sFound compiled %s in cache" % (progress, pyx_file)) - os.utime(fingerprint_file, None) - g = gzip_open(fingerprint_file, 'rb') - try: - f = open(c_file, 'wb') - try: - shutil.copyfileobj(g, f) - finally: - f.close() - finally: - g.close() + print(u"%sFound compiled %s in cache" % (progress, pyx_file)) + if os.path.exists(gz_fingerprint_file): + os.utime(gz_fingerprint_file, None) + with contextlib.closing(gzip_open(gz_fingerprint_file, 'rb')) as g: + with contextlib.closing(open(c_file, 'wb')) as f: + shutil.copyfileobj(g, f) + else: + os.utime(zip_fingerprint_file, None) + dirname = os.path.dirname(c_file) + with contextlib.closing(zipfile.ZipFile(zip_fingerprint_file)) as z: + for artifact in z.namelist(): + z.extract(artifact, os.path.join(dirname, artifact)) return if not quiet: - print("%sCythonizing %s" % (progress, pyx_file)) + print(u"%sCythonizing %s" % (progress, Utils.decode_filename(pyx_file))) if options is None: options = CompilationOptions(default_options) options.output_file = c_file options.embedded_metadata = embedded_metadata + old_warning_level = Errors.LEVEL + if show_all_warnings: + Errors.LEVEL = 0 + any_failures = 0 try: - result = compile([pyx_file], options) + result = compile_single(pyx_file, options, full_module_name=full_module_name) if result.num_errors > 0: any_failures = 1 except (EnvironmentError, PyrexError) as e: @@ -1042,21 +1301,31 @@ import traceback traceback.print_exc() any_failures = 1 + finally: + if show_all_warnings: + Errors.LEVEL = old_warning_level + if any_failures: if raise_on_failure: raise CompileError(None, pyx_file) elif os.path.exists(c_file): os.remove(c_file) elif fingerprint: - f = open(c_file, 'rb') - try: - g = gzip_open(fingerprint_file, 'wb') - try: - shutil.copyfileobj(f, g) - finally: - g.close() - finally: - f.close() + artifacts = list(filter(None, [ + getattr(result, attr, None) + for attr in ('c_file', 'h_file', 'api_file', 'i_file')])) + if len(artifacts) == 1: + fingerprint_file = gz_fingerprint_file + with contextlib.closing(open(c_file, 'rb')) as f: + with contextlib.closing(gzip_open(fingerprint_file + '.tmp', 'wb')) as g: + shutil.copyfileobj(f, g) + else: + fingerprint_file = zip_fingerprint_file + with contextlib.closing(zipfile.ZipFile( + fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: + for artifact in artifacts: + zip.write(artifact, os.path.basename(artifact)) + os.rename(fingerprint_file + '.tmp', fingerprint_file) def cythonize_one_helper(m): @@ -1077,9 +1346,10 @@ def cleanup_cache(cache, target_size, ratio=.85): try: p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) + stdout, _ = p.communicate() res = p.wait() if res == 0: - total_size = 1024 * int(p.stdout.read().strip().split()[0]) + total_size = 1024 * int(stdout.strip().split()[0]) if total_size < target_size: return except (OSError, ValueError): diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Inline.py cython-0.20.1+1~202203241016-9537/Cython/Build/Inline.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Inline.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Inline.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,32 +1,32 @@ from __future__ import absolute_import -import sys, os, re, inspect -import imp - -try: - import hashlib -except ImportError: - import md5 as hashlib +import hashlib +import inspect +import os +import re +import sys from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext import Cython -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler.Main import Context +from ..Compiler.Options import default_options -from ..Compiler.ParseTreeTransforms import (CythonTransform, - SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform) +from ..Compiler.Visitor import CythonTransform, EnvTransform +from ..Compiler.ParseTreeTransforms import SkipDeclarations from ..Compiler.TreeFragment import parse_from_strings from ..Compiler.StringEncoding import _unicode from .Dependencies import strip_string_literals, cythonize, cached_function -from ..Compiler import Pipeline, Nodes +from ..Compiler import Pipeline from ..Utils import get_cython_cache_dir import cython as cython_module -IS_PY3 = sys.version_info >= (3, 0) + +IS_PY3 = sys.version_info >= (3,) # A utility function to convert user-supplied ASCII strings to unicode. -if sys.version_info[0] < 3: +if not IS_PY3: def to_unicode(s): if isinstance(s, bytes): return s.decode('ascii') @@ -36,6 +36,20 @@ to_unicode = lambda x: x +if sys.version_info < (3, 5): + import imp + def load_dynamic(name, module_path): + return imp.load_dynamic(name, module_path) +else: + import importlib.util as _importlib_util + def load_dynamic(name, module_path): + spec = _importlib_util.spec_from_file_location(name, module_path) + module = _importlib_util.module_from_spec(spec) + # sys.modules[name] = module + spec.loader.exec_module(module) + return module + + class UnboundSymbols(EnvTransform, SkipDeclarations): def __init__(self): CythonTransform.__init__(self, None) @@ -90,7 +104,7 @@ elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray): return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim) else: - for base_type in py_type.mro(): + for base_type in py_type.__mro__: if base_type.__module__ in ('__builtin__', 'builtins'): return 'object' module = context.find_module(base_type.__module__, need_pxd=False) @@ -120,6 +134,7 @@ _cython_inline_cache = {} _cython_inline_default_context = _create_context(('.',)) + def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): for symbol in unbound_symbols: if symbol not in kwds: @@ -136,20 +151,37 @@ else: print("Couldn't find %r" % symbol) -def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), - cython_include_dirs=None, force=False, quiet=False, locals=None, globals=None, **kwds): + +def _inline_key(orig_code, arg_sigs, language_level): + key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__ + return hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest() + + +def cython_inline(code, get_type=unsafe_type, + lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), + cython_include_dirs=None, cython_compiler_directives=None, + force=False, quiet=False, locals=None, globals=None, language_level=None, **kwds): if get_type is None: get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context + cython_compiler_directives = dict(cython_compiler_directives) if cython_compiler_directives else {} + if language_level is None and 'language_level' not in cython_compiler_directives: + language_level = '3str' + if language_level is not None: + cython_compiler_directives['language_level'] = language_level + + key_hash = None + # Fast path if this has been called in this session. _unbound_symbols = _cython_inline_cache.get(code) if _unbound_symbols is not None: _populate_unbound(kwds, _unbound_symbols, locals, globals) args = sorted(kwds.items()) arg_sigs = tuple([(get_type(value, ctx), arg) for arg, value in args]) - invoke = _cython_inline_cache.get((code, arg_sigs)) + key_hash = _inline_key(code, arg_sigs, language_level) + invoke = _cython_inline_cache.get((code, arg_sigs, key_hash)) if invoke is not None: arg_list = [arg[1] for arg in args] return invoke(*arg_list) @@ -169,6 +201,7 @@ if not quiet: # Parsing from strings not fully supported (e.g. cimports). print("Could not parse code as a string (to extract unbound symbols).") + cimports = [] for name, arg in list(kwds.items()): if arg is cython_module: @@ -176,8 +209,9 @@ del kwds[name] arg_names = sorted(kwds) arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) - key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__ - module_name = "_cython_inline_" + hashlib.md5(_unicode(key).encode('utf-8')).hexdigest() + if key_hash is None: + key_hash = _inline_key(orig_code, arg_sigs, language_level) + module_name = "_cython_inline_" + key_hash if module_name in sys.modules: module = sys.modules[module_name] @@ -195,6 +229,7 @@ os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] + define_macros = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: @@ -205,6 +240,7 @@ if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) + define_macros.append(("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) @@ -227,23 +263,30 @@ finally: fh.close() extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - extra_compile_args = cflags) + name=module_name, + sources=[pyx_file], + include_dirs=c_include_dirs or None, + extra_compile_args=cflags or None, + define_macros=define_macros or None, + ) if build_extension is None: build_extension = _get_build_extension() - build_extension.extensions = cythonize([extension], include_path=cython_include_dirs or ['.'], quiet=quiet) + build_extension.extensions = cythonize( + [extension], + include_path=cython_include_dirs or ['.'], + compiler_directives=cython_compiler_directives, + quiet=quiet) build_extension.build_temp = os.path.dirname(pyx_file) build_extension.build_lib = lib_dir build_extension.run() - module = imp.load_dynamic(module_name, module_path) + module = load_dynamic(module_name, module_path) - _cython_inline_cache[orig_code, arg_sigs] = module.__invoke + _cython_inline_cache[orig_code, arg_sigs, key_hash] = module.__invoke arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list) + # Cached suffix used by cython_inline above. None should get # overridden with actual value upon the first cython_inline invocation cython_inline.so_ext = None @@ -288,37 +331,6 @@ return '\n'.join(module), ' ' + '\n '.join(function) -try: - from inspect import getcallargs -except ImportError: - def getcallargs(func, *arg_values, **kwd_values): - all = {} - args, varargs, kwds, defaults = inspect.getargspec(func) - if varargs is not None: - all[varargs] = arg_values[len(args):] - for name, value in zip(args, arg_values): - all[name] = value - for name, value in list(kwd_values.items()): - if name in args: - if name in all: - raise TypeError("Duplicate argument %s" % name) - all[name] = kwd_values.pop(name) - if kwds is not None: - all[kwds] = kwd_values - elif kwd_values: - raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values)) - if defaults is None: - defaults = () - first_default = len(args) - len(defaults) - for ix, name in enumerate(args): - if name not in all: - if ix >= first_default: - all[name] = defaults[ix - first_default] - else: - raise TypeError("Missing argument: %s" % name) - return all - - def get_body(source): ix = source.index(':') if source[:5] == 'lambda': @@ -336,7 +348,7 @@ self._body = get_body(inspect.getsource(f)) def __call__(self, *args, **kwds): - all = getcallargs(self._f, *args, **kwds) + all = inspect.getcallargs(self._f, *args, **kwds) if IS_PY3: return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all) else: diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/ipython-COPYING.rst cython-0.20.1+1~202203241016-9537/Cython/Build/ipython-COPYING.rst --- cython-0.20.1+1~201611251650-6686/Cython/Build/ipython-COPYING.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/ipython-COPYING.rst 2022-03-24 10:16:46.000000000 +0000 @@ -65,7 +65,7 @@ their copyright in the commit message of the change, when they commit the change to one of the IPython repositories. -With this in mind, the following banner should be used in any source code file +With this in mind, the following banner should be used in any source code file to indicate the copyright and license terms: :: diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/IpythonMagic.py cython-0.20.1+1~202203241016-9537/Cython/Build/IpythonMagic.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/IpythonMagic.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/IpythonMagic.py 2022-03-24 10:16:46.000000000 +0000 @@ -14,7 +14,7 @@ Usage ===== -To enable the magics below, execute ``%load_ext cythonmagic``. +To enable the magics below, execute ``%load_ext cython``. ``%%cython`` @@ -52,24 +52,20 @@ import re import sys import time +import copy +import distutils.log +import textwrap -try: - reload -except NameError: # Python 3 - from imp import reload - -try: - import hashlib -except ImportError: - import md5 as hashlib +IO_ENCODING = sys.getfilesystemencoding() +IS_PY2 = sys.version_info[0] < 3 +import hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext from IPython.core import display from IPython.core import magic_arguments from IPython.core.magic import Magics, magics_class, cell_magic -from IPython.utils import py3compat try: from IPython.paths import get_ipython_cache_dir except ImportError: @@ -81,21 +77,53 @@ from ..Compiler.Errors import CompileError from .Inline import cython_inline from .Dependencies import cythonize +from ..Utils import captured_fd, print_captured + + +PGO_CONFIG = { + 'gcc': { + 'gen': ['-fprofile-generate', '-fprofile-dir={TEMPDIR}'], + 'use': ['-fprofile-use', '-fprofile-correction', '-fprofile-dir={TEMPDIR}'], + }, + # blind copy from 'configure' script in CPython 3.7 + 'icc': { + 'gen': ['-prof-gen'], + 'use': ['-prof-use'], + } +} +PGO_CONFIG['mingw32'] = PGO_CONFIG['gcc'] + + +if IS_PY2: + def encode_fs(name): + return name if isinstance(name, bytes) else name.encode(IO_ENCODING) +else: + def encode_fs(name): + return name @magics_class class CythonMagics(Magics): def __init__(self, shell): - super(CythonMagics,self).__init__(shell) + super(CythonMagics, self).__init__(shell) self._reloads = {} self._code_cache = {} self._pyximport_installed = False def _import_all(self, module): - for k,v in module.__dict__.items(): - if not k.startswith('__'): - self.shell.push({k:v}) + mdict = module.__dict__ + if '__all__' in mdict: + keys = mdict['__all__'] + else: + keys = [k for k in mdict if not k.startswith('_')] + + for k in keys: + try: + self.shell.push({k: mdict[k]}) + except KeyError: + msg = "'module' object has no attribute '%s'" % k + raise AttributeError(msg) @cell_magic def cython_inline(self, line, cell): @@ -139,11 +167,15 @@ f.write(cell) if 'pyximport' not in sys.modules or not self._pyximport_installed: import pyximport - pyximport.install(reload_support=True) + pyximport.install() self._pyximport_installed = True if module_name in self._reloads: module = self._reloads[module_name] - reload(module) + # Note: reloading extension modules is not actually supported + # (requires PEP-489 reinitialisation support). + # Don't know why this should ever have worked as it reads here. + # All we really need to do is to update the globals below. + #reload(module) else: __import__(module_name) module = sys.modules[module_name] @@ -152,6 +184,19 @@ @magic_arguments.magic_arguments() @magic_arguments.argument( + '-a', '--annotate', action='store_const', const='default', dest='annotate', + help="Produce a colorized HTML version of the source." + ) + @magic_arguments.argument( + '--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help="Produce a colorized HTML version of the source " + "which includes entire generated C/C++-code." + ) + @magic_arguments.argument( + '-+', '--cplus', action='store_true', default=False, + help="Output a C++ rather than C file." + ) + @magic_arguments.argument( '-3', dest='language_level', action='store_const', const=3, default=None, help="Select Python 3 syntax." ) @@ -160,6 +205,11 @@ help="Select Python 2 syntax." ) @magic_arguments.argument( + '-f', '--force', action='store_true', default=False, + help="Force the compilation of a new module, even if the source has been " + "previously compiled." + ) + @magic_arguments.argument( '-c', '--compile-args', action='append', default=[], help="Extra flags to pass to compiler via the `extra_compile_args` " "Extension flag (can be specified multiple times)." @@ -189,17 +239,19 @@ "multiple times)." ) @magic_arguments.argument( - '-+', '--cplus', action='store_true', default=False, - help="Output a C++ rather than C file." + '-S', '--src', action='append', default=[], + help="Add a path to the list of src files (can be specified " + "multiple times)." ) @magic_arguments.argument( - '-f', '--force', action='store_true', default=False, - help="Force the compilation of a new module, even if the source has been " - "previously compiled." + '--pgo', dest='pgo', action='store_true', default=False, + help=("Enable profile guided optimisation in the C compiler. " + "Compiles the cell twice and executes it in between to generate a runtime profile.") ) @magic_arguments.argument( - '-a', '--annotate', action='store_true', default=False, - help="Produce a colorized HTML version of the source." + '--verbose', dest='quiet', action='store_false', default=True, + help=("Print debug information like generated .c/.cpp file location " + "and exact gcc/g++ command invoked.") ) @cell_magic def cython(self, line, cell): @@ -221,76 +273,90 @@ %%cython --compile-args=-fopenmp --link-args=-fopenmp ... + + To enable profile guided optimisation, pass the ``--pgo`` option. + Note that the cell itself needs to take care of establishing a suitable + profile when executed. This can be done by implementing the functions to + optimise, and then calling them directly in the same cell on some realistic + training data like this:: + + %%cython --pgo + def critical_function(data): + for item in data: + ... + + # execute function several times to build profile + from somewhere import some_typical_data + for _ in range(100): + critical_function(some_typical_data) + + In Python 3.5 and later, you can distinguish between the profile and + non-profile runs as follows:: + + if "_pgo_" in __name__: + ... # execute critical code here """ args = magic_arguments.parse_argstring(self.cython, line) - code = cell if cell.endswith('\n') else cell+'\n' + code = cell if cell.endswith('\n') else cell + '\n' lib_dir = os.path.join(get_ipython_cache_dir(), 'cython') - quiet = True - key = code, line, sys.version_info, sys.executable, cython_version + key = (code, line, sys.version_info, sys.executable, cython_version) if not os.path.exists(lib_dir): os.makedirs(lib_dir) + if args.pgo: + key += ('pgo',) if args.force: # Force a new module name by adding the current time to the # key which is hashed to determine the module name. - key += time.time(), + key += (time.time(),) if args.name: - module_name = py3compat.unicode_to_str(args.name) + module_name = str(args.name) # no-op in Py3 else: - module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() + module_name = "_cython_magic_" + hashlib.sha1(str(key).encode('utf-8')).hexdigest() + html_file = os.path.join(lib_dir, module_name + '.html') module_path = os.path.join(lib_dir, module_name + self.so_ext) have_module = os.path.isfile(module_path) - need_cythonize = not have_module + need_cythonize = args.pgo or not have_module if args.annotate: - html_file = os.path.join(lib_dir, module_name + '.html') if not os.path.isfile(html_file): need_cythonize = True + extension = None if need_cythonize: - c_include_dirs = args.include - if 'numpy' in code: - import numpy - c_include_dirs.append(numpy.get_include()) - pyx_file = os.path.join(lib_dir, module_name + '.pyx') - pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding()) - with io.open(pyx_file, 'w', encoding='utf-8') as f: - f.write(code) - extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - library_dirs = args.library_dirs, - extra_compile_args = args.compile_args, - extra_link_args = args.link_args, - libraries = args.lib, - language = 'c++' if args.cplus else 'c', - ) - build_extension = self._get_build_extension() - try: - opts = dict( - quiet=quiet, - annotate=args.annotate, - force=True, - ) - if args.language_level is not None: - assert args.language_level in (2, 3) - opts['language_level'] = args.language_level - elif sys.version_info[0] > 2: - opts['language_level'] = 3 - build_extension.extensions = cythonize([extension], **opts) - except CompileError: - return - - if not have_module: - build_extension.build_temp = os.path.dirname(pyx_file) - build_extension.build_lib = lib_dir - build_extension.run() + extensions = self._cythonize(module_name, code, lib_dir, args, quiet=args.quiet) + if extensions is None: + # Compilation failed and printed error message + return None + assert len(extensions) == 1 + extension = extensions[0] self._code_cache[key] = module_name + if args.pgo: + self._profile_pgo_wrapper(extension, lib_dir) + + def print_compiler_output(stdout, stderr, where): + # On windows, errors are printed to stdout, we redirect both to sys.stderr. + print_captured(stdout, where, u"Content of stdout:\n") + print_captured(stderr, where, u"Content of stderr:\n") + + get_stderr = get_stdout = None + try: + with captured_fd(1) as get_stdout: + with captured_fd(2) as get_stderr: + self._build_extension( + extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet) + except (distutils.errors.CompileError, distutils.errors.LinkError): + # Build failed, print error message from compiler/linker + print_compiler_output(get_stdout(), get_stderr(), sys.stderr) + return None + + # Build seems ok, but we might still want to show any warnings that occurred + print_compiler_output(get_stdout(), get_stderr(), sys.stdout) + module = imp.load_dynamic(module_name, module_path) self._import_all(module) @@ -309,6 +375,128 @@ else: return display.HTML(self.clean_annotated_html(annotated_html)) + def _profile_pgo_wrapper(self, extension, lib_dir): + """ + Generate a .c file for a separate extension module that calls the + module init function of the original module. This makes sure that the + PGO profiler sees the correct .o file of the final module, but it still + allows us to import the module under a different name for profiling, + before recompiling it into the PGO optimised module. Overwriting and + reimporting the same shared library is not portable. + """ + extension = copy.copy(extension) # shallow copy, do not modify sources in place! + module_name = extension.name + pgo_module_name = '_pgo_' + module_name + pgo_wrapper_c_file = os.path.join(lib_dir, pgo_module_name + '.c') + with io.open(pgo_wrapper_c_file, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(u""" + #include "Python.h" + #if PY_MAJOR_VERSION < 3 + extern PyMODINIT_FUNC init%(module_name)s(void); + PyMODINIT_FUNC init%(pgo_module_name)s(void); /*proto*/ + PyMODINIT_FUNC init%(pgo_module_name)s(void) { + PyObject *sys_modules; + init%(module_name)s(); if (PyErr_Occurred()) return; + sys_modules = PyImport_GetModuleDict(); /* borrowed, no exception, "never" fails */ + if (sys_modules) { + PyObject *module = PyDict_GetItemString(sys_modules, "%(module_name)s"); if (!module) return; + PyDict_SetItemString(sys_modules, "%(pgo_module_name)s", module); + Py_DECREF(module); + } + } + #else + extern PyMODINIT_FUNC PyInit_%(module_name)s(void); + PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void); /*proto*/ + PyMODINIT_FUNC PyInit_%(pgo_module_name)s(void) { + return PyInit_%(module_name)s(); + } + #endif + """ % {'module_name': module_name, 'pgo_module_name': pgo_module_name})) + + extension.sources = extension.sources + [pgo_wrapper_c_file] # do not modify in place! + extension.name = pgo_module_name + + self._build_extension(extension, lib_dir, pgo_step_name='gen') + + # import and execute module code to generate profile + so_module_path = os.path.join(lib_dir, pgo_module_name + self.so_ext) + imp.load_dynamic(pgo_module_name, so_module_path) + + def _cythonize(self, module_name, code, lib_dir, args, quiet=True): + pyx_file = os.path.join(lib_dir, module_name + '.pyx') + pyx_file = encode_fs(pyx_file) + + c_include_dirs = args.include + c_src_files = list(map(str, args.src)) + if 'numpy' in code: + import numpy + c_include_dirs.append(numpy.get_include()) + with io.open(pyx_file, 'w', encoding='utf-8') as f: + f.write(code) + extension = Extension( + name=module_name, + sources=[pyx_file] + c_src_files, + include_dirs=c_include_dirs, + library_dirs=args.library_dirs, + extra_compile_args=args.compile_args, + extra_link_args=args.link_args, + libraries=args.lib, + language='c++' if args.cplus else 'c', + ) + try: + opts = dict( + quiet=quiet, + annotate=args.annotate, + force=True, + language_level=min(3, sys.version_info[0]), + ) + if args.language_level is not None: + assert args.language_level in (2, 3) + opts['language_level'] = args.language_level + return cythonize([extension], **opts) + except CompileError: + return None + + def _build_extension(self, extension, lib_dir, temp_dir=None, pgo_step_name=None, quiet=True): + build_extension = self._get_build_extension( + extension, lib_dir=lib_dir, temp_dir=temp_dir, pgo_step_name=pgo_step_name) + old_threshold = None + try: + if not quiet: + old_threshold = distutils.log.set_threshold(distutils.log.DEBUG) + build_extension.run() + finally: + if not quiet and old_threshold is not None: + distutils.log.set_threshold(old_threshold) + + def _add_pgo_flags(self, build_extension, step_name, temp_dir): + compiler_type = build_extension.compiler.compiler_type + if compiler_type == 'unix': + compiler_cmd = build_extension.compiler.compiler_so + # TODO: we could try to call "[cmd] --version" for better insights + if not compiler_cmd: + pass + elif 'clang' in compiler_cmd or 'clang' in compiler_cmd[0]: + compiler_type = 'clang' + elif 'icc' in compiler_cmd or 'icc' in compiler_cmd[0]: + compiler_type = 'icc' + elif 'gcc' in compiler_cmd or 'gcc' in compiler_cmd[0]: + compiler_type = 'gcc' + elif 'g++' in compiler_cmd or 'g++' in compiler_cmd[0]: + compiler_type = 'gcc' + config = PGO_CONFIG.get(compiler_type) + orig_flags = [] + if config and step_name in config: + flags = [f.format(TEMPDIR=temp_dir) for f in config[step_name]] + for extension in build_extension.extensions: + orig_flags.append((extension.extra_compile_args, extension.extra_link_args)) + extension.extra_compile_args = extension.extra_compile_args + flags + extension.extra_link_args = extension.extra_link_args + flags + else: + print("No PGO %s configuration known for C compiler type '%s'" % (step_name, compiler_type), + file=sys.stderr) + return orig_flags + @property def so_ext(self): """The extension suffix for compiled modules.""" @@ -330,7 +518,8 @@ else: _path_created.clear() - def _get_build_extension(self): + def _get_build_extension(self, extension=None, lib_dir=None, temp_dir=None, + pgo_step_name=None, _build_ext=build_ext): self._clear_distutils_mkpath_cache() dist = Distribution() config_files = dist.find_config_files() @@ -339,8 +528,28 @@ except ValueError: pass dist.parse_config_files(config_files) - build_extension = build_ext(dist) + + if not temp_dir: + temp_dir = lib_dir + add_pgo_flags = self._add_pgo_flags + + if pgo_step_name: + base_build_ext = _build_ext + class _build_ext(_build_ext): + def build_extensions(self): + add_pgo_flags(self, pgo_step_name, temp_dir) + base_build_ext.build_extensions(self) + + build_extension = _build_ext(dist) build_extension.finalize_options() + if temp_dir: + temp_dir = encode_fs(temp_dir) + build_extension.build_temp = temp_dir + if lib_dir: + lib_dir = encode_fs(lib_dir) + build_extension.build_lib = lib_dir + if extension is not None: + build_extension.extensions = [extension] return build_extension @staticmethod @@ -355,10 +564,10 @@ return html __doc__ = __doc__.format( - # rST doesn't see the -+ flag as part of an option list, so we - # hide it from the module-level docstring. - CYTHON_DOC = dedent(CythonMagics.cython.__doc__\ - .replace('-+, --cplus','--cplus ')), - CYTHON_INLINE_DOC = dedent(CythonMagics.cython_inline.__doc__), - CYTHON_PYXIMPORT_DOC = dedent(CythonMagics.cython_pyximport.__doc__), + # rST doesn't see the -+ flag as part of an option list, so we + # hide it from the module-level docstring. + CYTHON_DOC=dedent(CythonMagics.cython.__doc__\ + .replace('-+, --cplus', '--cplus ')), + CYTHON_INLINE_DOC=dedent(CythonMagics.cython_inline.__doc__), + CYTHON_PYXIMPORT_DOC=dedent(CythonMagics.cython_pyximport.__doc__), ) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestCyCache.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestCyCache.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestCyCache.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestCyCache.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,116 @@ +import difflib +import glob +import gzip +import os +import tempfile + +import Cython.Build.Dependencies +import Cython.Utils +from Cython.TestUtils import CythonTest + + +class TestCyCache(CythonTest): + + def setUp(self): + CythonTest.setUp(self) + self.temp_dir = tempfile.mkdtemp( + prefix='cycache-test', + dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None) + self.src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + self.cache_dir = tempfile.mkdtemp(prefix='cache', dir=self.temp_dir) + + def cache_files(self, file_glob): + return glob.glob(os.path.join(self.cache_dir, file_glob)) + + def fresh_cythonize(self, *args, **kwargs): + Cython.Utils.clear_function_caches() + Cython.Build.Dependencies._dep_tree = None # discard method caches + Cython.Build.Dependencies.cythonize(*args, **kwargs) + + def test_cycache_switch(self): + content1 = 'value = 1\n' + content2 = 'value = 2\n' + a_pyx = os.path.join(self.src_dir, 'a.pyx') + a_c = a_pyx[:-4] + '.c' + + with open(a_pyx, 'w') as f: + f.write(content1) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + self.assertEqual(1, len(self.cache_files('a.c*'))) + with open(a_c) as f: + a_contents1 = f.read() + os.unlink(a_c) + + with open(a_pyx, 'w') as f: + f.write(content2) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + with open(a_c) as f: + a_contents2 = f.read() + os.unlink(a_c) + + self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!') + self.assertEqual(2, len(self.cache_files('a.c*'))) + + with open(a_pyx, 'w') as f: + f.write(content1) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + self.assertEqual(2, len(self.cache_files('a.c*'))) + with open(a_c) as f: + a_contents = f.read() + self.assertEqual( + a_contents, a_contents1, + msg='\n'.join(list(difflib.unified_diff( + a_contents.split('\n'), a_contents1.split('\n')))[:10])) + + def test_cycache_uses_cache(self): + a_pyx = os.path.join(self.src_dir, 'a.pyx') + a_c = a_pyx[:-4] + '.c' + with open(a_pyx, 'w') as f: + f.write('pass') + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0]) + gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii')) + os.unlink(a_c) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + with open(a_c) as f: + a_contents = f.read() + self.assertEqual(a_contents, 'fake stuff', + 'Unexpected contents: %s...' % a_contents[:100]) + + def test_multi_file_output(self): + a_pyx = os.path.join(self.src_dir, 'a.pyx') + a_c = a_pyx[:-4] + '.c' + a_h = a_pyx[:-4] + '.h' + a_api_h = a_pyx[:-4] + '_api.h' + with open(a_pyx, 'w') as f: + f.write('cdef public api int foo(int x): return x\n') + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + expected = [a_c, a_h, a_api_h] + for output in expected: + self.assertTrue(os.path.exists(output), output) + os.unlink(output) + self.fresh_cythonize(a_pyx, cache=self.cache_dir) + for output in expected: + self.assertTrue(os.path.exists(output), output) + + def test_options_invalidation(self): + hash_pyx = os.path.join(self.src_dir, 'options.pyx') + hash_c = hash_pyx[:-len('.pyx')] + '.c' + + with open(hash_pyx, 'w') as f: + f.write('pass') + self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False) + self.assertEqual(1, len(self.cache_files('options.c*'))) + + os.unlink(hash_c) + self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=True) + self.assertEqual(2, len(self.cache_files('options.c*'))) + + os.unlink(hash_c) + self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=False) + self.assertEqual(2, len(self.cache_files('options.c*'))) + + os.unlink(hash_c) + self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=True) + self.assertEqual(2, len(self.cache_files('options.c*'))) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestCythonizeArgsParser.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestCythonizeArgsParser.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestCythonizeArgsParser.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestCythonizeArgsParser.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,482 @@ +from Cython.Build.Cythonize import ( + create_args_parser, parse_args_raw, parse_args, + parallel_compiles +) + +from Cython.Compiler import Options +from Cython.Compiler.Tests.Utils import backup_Options, restore_Options, check_global_options + +from unittest import TestCase + +import sys +try: + from StringIO import StringIO +except ImportError: + from io import StringIO # doesn't accept 'str' in Py2 + + +class TestCythonizeArgsParser(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.parse_args = lambda x, parser=create_args_parser() : parse_args_raw(parser, x) + + + def are_default(self, options, skip): + # empty containers + empty_containers = ['directives', 'compile_time_env', 'options', 'excludes'] + are_none = ['language_level', 'annotate', 'build', 'build_inplace', 'force', 'quiet', 'lenient', 'keep_going', 'no_docstrings'] + for opt_name in empty_containers: + if len(getattr(options, opt_name))!=0 and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + for opt_name in are_none: + if (getattr(options, opt_name) is not None) and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + if options.parallel!=parallel_compiles and ('parallel' not in skip): + return False + return True + + # testing directives: + def test_directive_short(self): + options, args = self.parse_args(['-X', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_long(self): + options, args = self.parse_args(['--directive', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_multiple(self): + options, args = self.parse_args(['-X', 'cdivision=True', '-X', 'c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_multiple_v2(self): + options, args = self.parse_args(['-X', 'cdivision=True,c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_value_yes(self): + options, args = self.parse_args(['-X', 'cdivision=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_value_no(self): + options, args = self.parse_args(['-X', 'cdivision=no']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], False) + + def test_directive_value_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision=sadfasd']) + + def test_directive_key_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'abracadabra']) + + def test_directive_no_value(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision']) + + def test_directives_types(self): + directives = { + 'auto_pickle': True, + 'c_string_type': 'bytearray', + 'c_string_type': 'bytes', + 'c_string_type': 'str', + 'c_string_type': 'bytearray', + 'c_string_type': 'unicode', + 'c_string_encoding' : 'ascii', + 'language_level' : 2, + 'language_level' : 3, + 'language_level' : '3str', + 'set_initial_path' : 'my_initial_path', + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + options, args = self.parse_args(['-X', cmd]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives']), msg = "Error for option: "+cmd) + self.assertEqual(options.directives[key], value, msg = "Error for option: "+cmd) + + def test_directives_wrong(self): + directives = { + 'auto_pickle': 42, # for bool type + 'auto_pickle': 'NONONO', # for bool type + 'c_string_type': 'bites', + #'c_string_encoding' : 'a', + #'language_level' : 4, + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + with self.assertRaises(ValueError, msg = "Error for option: "+cmd) as context: + options, args = self.parse_args(['-X', cmd]) + + def test_compile_time_env_short(self): + options, args = self.parse_args(['-E', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_long(self): + options, args = self.parse_args(['--compile-time-env', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_multiple(self): + options, args = self.parse_args(['-E', 'MYSIZE=10', '-E', 'ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + def test_compile_time_env_multiple_v2(self): + options, args = self.parse_args(['-E', 'MYSIZE=10,ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + #testing options + def test_option_short(self): + options, args = self.parse_args(['-s', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_long(self): + options, args = self.parse_args(['--option', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_multiple(self): + options, args = self.parse_args(['-s', 'docstrings=True', '-s', 'buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_multiple_v2(self): + options, args = self.parse_args(['-s', 'docstrings=True,buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_value_yes(self): + options, args = self.parse_args(['-s', 'docstrings=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_4242(self): + options, args = self.parse_args(['-s', 'docstrings=4242']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_0(self): + options, args = self.parse_args(['-s', 'docstrings=0']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_value_emptystr(self): + options, args = self.parse_args(['-s', 'docstrings=']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_a_str(self): + options, args = self.parse_args(['-s', 'docstrings=BB']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_no(self): + options, args = self.parse_args(['-s', 'docstrings=nO']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_no_value(self): + options, args = self.parse_args(['-s', 'docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_any_key(self): + options, args = self.parse_args(['-s', 'abracadabra']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['abracadabra'], True) + + def test_language_level_2(self): + options, args = self.parse_args(['-2']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 2) + + def test_language_level_3(self): + options, args = self.parse_args(['-3']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 3) + + def test_language_level_3str(self): + options, args = self.parse_args(['--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, '3str') + + def test_annotate_short(self): + options, args = self.parse_args(['-a']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_long(self): + options, args = self.parse_args(['--annotate']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_fullc(self): + options, args = self.parse_args(['--annotate-fullc']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'fullc') + + def test_annotate_and_positional(self): + options, args = self.parse_args(['-a', 'foo.pyx']) + self.assertEqual(args, ['foo.pyx']) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_and_optional(self): + options, args = self.parse_args(['-a', '--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate', 'language_level'])) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.language_level, '3str') + + def test_exclude_short(self): + options, args = self.parse_args(['-x', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_long(self): + options, args = self.parse_args(['--exclude', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_multiple(self): + options, args = self.parse_args(['--exclude', '*.pyx', '--exclude', '*.py', ]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertEqual(options.excludes, ['*.pyx', '*.py']) + + def test_build_short(self): + options, args = self.parse_args(['-b']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_build_long(self): + options, args = self.parse_args(['--build']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_inplace_short(self): + options, args = self.parse_args(['-i']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_inplace_long(self): + options, args = self.parse_args(['--inplace']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_parallel_short(self): + options, args = self.parse_args(['-j', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_parallel_long(self): + options, args = self.parse_args(['--parallel', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_force_short(self): + options, args = self.parse_args(['-f']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_force_long(self): + options, args = self.parse_args(['--force']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_quite_short(self): + options, args = self.parse_args(['-q']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_quite_long(self): + options, args = self.parse_args(['--quiet']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_lenient_long(self): + options, args = self.parse_args(['--lenient']) + self.assertTrue(self.are_default(options, ['lenient'])) + self.assertFalse(args) + self.assertEqual(options.lenient, True) + + def test_keep_going_short(self): + options, args = self.parse_args(['-k']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_keep_going_long(self): + options, args = self.parse_args(['--keep-going']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_no_docstrings_long(self): + options, args = self.parse_args(['--no-docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['no_docstrings'])) + self.assertEqual(options.no_docstrings, True) + + def test_file_name(self): + options, args = self.parse_args(['file1.pyx', 'file2.pyx']) + self.assertEqual(len(args), 2) + self.assertEqual(args[0], 'file1.pyx') + self.assertEqual(args[1], 'file2.pyx') + self.assertTrue(self.are_default(options, [])) + + def test_option_first(self): + options, args = self.parse_args(['-i', 'file.pyx']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_file_inbetween(self): + options, args = self.parse_args(['-i', 'file.pyx', '-a']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['build_inplace', 'annotate'])) + + def test_option_trailing(self): + options, args = self.parse_args(['file.pyx', '-i']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_interspersed_positional(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional2(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx', '-a', 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional3(self): + options, sources = self.parse_args([ + '-f', 'f1', 'f2', '-a', + 'f3', 'f4', '-a', 'f5' + ]) + self.assertEqual(sources, ['f1', 'f2', 'f3', 'f4', 'f5']) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.force, True) + self.assertTrue(self.are_default(options, ['annotate', 'force'])) + + def test_wrong_option(self): + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + self.assertRaises(SystemExit, self.parse_args, + ['--unknown-option'] + ) + finally: + sys.stderr = old_stderr + self.assertTrue(stderr.getvalue()) + + +class TestParseArgs(TestCase): + def setUp(self): + self._options_backup = backup_Options() + + def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def test_build_set_for_inplace(self): + options, args = parse_args(['foo.pyx', '-i']) + self.assertEqual(options.build, True) + self.check_default_global_options() + + def test_lenient(self): + options, sources = parse_args(['foo.pyx', '--lenient']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.error_on_unknown_names, False) + self.assertEqual(Options.error_on_uninitialized, False) + self.check_default_global_options(['error_on_unknown_names', 'error_on_uninitialized']) + + def test_annotate(self): + options, sources = parse_args(['foo.pyx', '--annotate']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + + def test_annotate_fullc(self): + options, sources = parse_args(['foo.pyx', '--annotate-fullc']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + + def test_no_docstrings(self): + options, sources = parse_args(['foo.pyx', '--no-docstrings']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.docstrings, False) + self.check_default_global_options(['docstrings']) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestInline.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestInline.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestInline.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestInline.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,6 @@ -import os, tempfile +import os +import tempfile +import unittest from Cython.Shadow import inline from Cython.Build.Inline import safe_type from Cython.TestUtils import CythonTest @@ -24,10 +26,10 @@ self.test_kwds['lib_dir'] = lib_dir def test_simple(self): - self.assertEquals(inline("return 1+2", **self.test_kwds), 3) + self.assertEqual(inline("return 1+2", **self.test_kwds), 3) def test_types(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" cimport cython return cython.typeof(a), cython.typeof(b) """, a=1.0, b=[], **self.test_kwds), ('double', 'list object')) @@ -35,13 +37,13 @@ def test_locals(self): a = 1 b = 2 - self.assertEquals(inline("return a+b", **self.test_kwds), 3) + self.assertEqual(inline("return a+b", **self.test_kwds), 3) def test_globals(self): - self.assertEquals(inline("return global_value + 1", **self.test_kwds), global_value + 1) + self.assertEqual(inline("return global_value + 1", **self.test_kwds), global_value + 1) def test_no_return(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" a = 1 cdef double b = 2 cdef c = [] @@ -49,7 +51,13 @@ def test_def_node(self): foo = inline("def foo(x): return x * x", **self.test_kwds)['foo'] - self.assertEquals(foo(7), 49) + self.assertEqual(foo(7), 49) + + def test_class_ref(self): + class Type(object): + pass + tp = inline("Type")['Type'] + self.assertEqual(tp, Type) def test_pure(self): import cython as cy @@ -58,13 +66,47 @@ c = cy.declare(cy.pointer(cy.float), &b) return b """, a=3, **self.test_kwds) - self.assertEquals(type(b), float) - - if has_numpy: + self.assertEqual(type(b), float) - def test_numpy(self): - import numpy - a = numpy.ndarray((10, 20)) - a[0,0] = 10 - self.assertEquals(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') - self.assertEquals(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) + def test_compiler_directives(self): + self.assertEqual( + inline('return sum(x)', + x=[1, 2, 3], + cython_compiler_directives={'boundscheck': False}), + 6 + ) + + def test_lang_version(self): + # GH-3419. Caching for inline code didn't always respect compiler directives. + inline_divcode = "def f(int a, int b): return a/b" + self.assertEqual( + inline(inline_divcode, language_level=2)['f'](5,2), + 2 + ) + self.assertEqual( + inline(inline_divcode, language_level=3)['f'](5,2), + 2.5 + ) + self.assertEqual( + inline(inline_divcode, language_level=2)['f'](5,2), + 2 + ) + + def test_repeated_use(self): + inline_mulcode = "def f(int a, int b): return a * b" + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + self.assertEqual(inline(inline_mulcode)['f'](5, 3), 15) + self.assertEqual(inline(inline_mulcode)['f'](6, 2), 12) + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + + f = inline(inline_mulcode)['f'] + self.assertEqual(f(5, 2), 10) + self.assertEqual(f(5, 3), 15) + + @unittest.skipIf(not has_numpy, "NumPy is not available") + def test_numpy(self): + import numpy + a = numpy.ndarray((10, 20)) + a[0,0] = 10 + self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') + self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestIpythonMagic.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestIpythonMagic.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestIpythonMagic.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestIpythonMagic.py 2022-03-24 10:16:46.000000000 +0000 @@ -3,37 +3,92 @@ """Tests for the Cython magics extension.""" +from __future__ import absolute_import + import os +import io import sys +from contextlib import contextmanager +from Cython.Build import IpythonMagic +from Cython.TestUtils import CythonTest +from Cython.Compiler.Annotate import AnnotationCCodeWriter try: - from IPython.testing.globalipapp import get_ipython - from IPython.utils import py3compat -except: - __test__ = False + import IPython.testing.globalipapp +except ImportError: + # Disable tests and fake helpers for initialisation below. + def skip_if_not_installed(_): + return None +else: + def skip_if_not_installed(c): + return c try: - # disable IPython history thread to avoid having to clean it up + # disable IPython history thread before it gets started to avoid having to clean it up from IPython.core.history import HistoryManager HistoryManager.enabled = False except ImportError: pass -from Cython.TestUtils import CythonTest -ip = get_ipython() -code = py3compat.str_to_unicode("""\ +@contextmanager +def capture_output(): + backup = sys.stdout, sys.stderr + try: + replacement = [ + io.TextIOWrapper(io.BytesIO(), encoding=sys.stdout.encoding), + io.TextIOWrapper(io.BytesIO(), encoding=sys.stderr.encoding), + ] + sys.stdout, sys.stderr = replacement + output = [] + yield output + finally: + sys.stdout, sys.stderr = backup + for wrapper in replacement: + wrapper.seek(0) # rewind + output.append(wrapper.read()) + wrapper.close() + + +code = u"""\ def f(x): return 2*x -""") +""" -cython3_code = py3compat.str_to_unicode("""\ +cython3_code = u"""\ def f(int x): return 2 / x def call(x): return f(*(x,)) -""") +""" + +pgo_cython3_code = cython3_code + u"""\ +def main(): + for _ in range(100): call(5) +main() +""" + +compile_error_code = u'''\ +cdef extern from *: + """ + xxx a=1; + """ + int a; +def doit(): + return a +''' + +compile_warning_code = u'''\ +cdef extern from *: + """ + #pragma message ( "CWarning" ) + int a = 42; + """ + int a; +def doit(): + return a +''' if sys.platform == 'win32': @@ -42,26 +97,40 @@ from unittest import skip as skip_win32 except ImportError: # poor dev's silent @unittest.skip() - def skip_win32(f): - return lambda self: None + def skip_win32(dummy): + def _skip_win32(func): + return None + return _skip_win32 else: - def skip_win32(f): - return f + def skip_win32(dummy): + def _skip_win32(func): + def wrapper(*args, **kwargs): + func(*args, **kwargs) + return wrapper + return _skip_win32 +@skip_if_not_installed class TestIPythonMagic(CythonTest): + @classmethod + def setUpClass(cls): + CythonTest.setUpClass() + cls._ip = IPython.testing.globalipapp.get_ipython() + def setUp(self): CythonTest.setUp(self) - ip.extension_manager.load_extension('cython') + self._ip.extension_manager.load_extension('cython') def test_cython_inline(self): + ip = self._ip ip.ex('a=10; b=20') result = ip.run_cell_magic('cython_inline', '', 'return a+b') self.assertEqual(result, 30) - @skip_win32 + @skip_win32('Skip on Windows') def test_cython_pyximport(self): + ip = self._ip module_name = '_test_cython_pyximport' ip.run_cell_magic('cython_pyximport', module_name, code) ip.ex('g = f(10)') @@ -75,12 +144,14 @@ pass def test_cython(self): + ip = self._ip ip.run_cell_magic('cython', '', code) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) def test_cython_name(self): # The Cython module named 'mymodule' defines the function f. + ip = self._ip ip.run_cell_magic('cython', '--name=mymodule', code) # This module can now be imported in the interactive namespace. ip.ex('import mymodule; g = mymodule.f(10)') @@ -88,6 +159,7 @@ def test_cython_language_level(self): # The Cython cell defines the functions f() and call(). + ip = self._ip ip.run_cell_magic('cython', '', cython3_code) ip.ex('g = f(10); h = call(10)') if sys.version_info[0] < 3: @@ -99,6 +171,7 @@ def test_cython3(self): # The Cython cell defines the functions f() and call(). + ip = self._ip ip.run_cell_magic('cython', '-3', cython3_code) ip.ex('g = f(10); h = call(10)') self.assertEqual(ip.user_ns['g'], 2.0 / 10.0) @@ -106,17 +179,129 @@ def test_cython2(self): # The Cython cell defines the functions f() and call(). + ip = self._ip ip.run_cell_magic('cython', '-2', cython3_code) ip.ex('g = f(10); h = call(10)') self.assertEqual(ip.user_ns['g'], 2 // 10) self.assertEqual(ip.user_ns['h'], 2 // 10) - @skip_win32 + def test_cython_compile_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3', compile_error_code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_link_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3 -l=xxxxxxxx', code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n!" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_warning_shown(self): + ip = self._ip + with capture_output() as out: + # force rebuild, otherwise no warning as after the first success + # no build step is performed + ip.run_cell_magic('cython', '-3 -f', compile_warning_code) + captured_out, captured_err = out + + # check that warning was printed to stdout even if build hasn't failed + self.assertTrue("CWarning" in captured_out) + + @skip_win32('Skip on Windows') + def test_cython3_pgo(self): + # The Cython cell defines the functions f() and call(). + ip = self._ip + ip.run_cell_magic('cython', '-3 --pgo', pgo_cython3_code) + ip.ex('g = f(10); h = call(10); main()') + self.assertEqual(ip.user_ns['g'], 2.0 / 10.0) + self.assertEqual(ip.user_ns['h'], 2.0 / 10.0) + + @skip_win32('Skip on Windows') def test_extlibs(self): - code = py3compat.str_to_unicode(""" + ip = self._ip + code = u""" from libc.math cimport sin x = sin(0.0) - """) + """ ip.user_ns['x'] = 1 ip.run_cell_magic('cython', '-l m', code) self.assertEqual(ip.user_ns['x'], 0) + + + def test_cython_verbose(self): + ip = self._ip + ip.run_cell_magic('cython', '--verbose', code) + ip.ex('g = f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + + def test_cython_verbose_thresholds(self): + @contextmanager + def mock_distutils(): + class MockLog: + DEBUG = 1 + INFO = 2 + thresholds = [INFO] + + def set_threshold(self, val): + self.thresholds.append(val) + return self.thresholds[-2] + + + new_log = MockLog() + old_log = IpythonMagic.distutils.log + try: + IpythonMagic.distutils.log = new_log + yield new_log + finally: + IpythonMagic.distutils.log = old_log + + ip = self._ip + with mock_distutils() as verbose_log: + ip.run_cell_magic('cython', '--verbose', code) + ip.ex('g = f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + self.assertEqual([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO], + verbose_log.thresholds) + + with mock_distutils() as normal_log: + ip.run_cell_magic('cython', '', code) + ip.ex('g = f(10)') + self.assertEqual(ip.user_ns['g'], 20.0) + self.assertEqual([normal_log.INFO], normal_log.thresholds) + + def test_cython_no_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '', code) + self.assertTrue(html is None) + + def test_cython_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_default(self): + ip = self._ip + html = ip.run_cell_magic('cython', '-a', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_complete_c_code(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate-fullc', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html.data) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestRecythonize.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestRecythonize.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestRecythonize.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestRecythonize.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,212 @@ +import shutil +import os +import tempfile +import time + +import Cython.Build.Dependencies +import Cython.Utils +from Cython.TestUtils import CythonTest + + +def fresh_cythonize(*args, **kwargs): + Cython.Utils.clear_function_caches() + Cython.Build.Dependencies._dep_tree = None # discard method caches + Cython.Build.Dependencies.cythonize(*args, **kwargs) + +class TestRecythonize(CythonTest): + + def setUp(self): + CythonTest.setUp(self) + self.temp_dir = ( + tempfile.mkdtemp( + prefix='recythonize-test', + dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None + ) + ) + + def tearDown(self): + CythonTest.tearDown(self) + shutil.rmtree(self.temp_dir) + + def test_recythonize_pyx_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.pyx" are "a.pxd" and "a.pyx". + self.assertEqual({a_pxd, a_pyx}, dep_tree.all_dependencies(a_pyx)) + + # Cythonize to create a.c + fresh_cythonize(a_pyx) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_pyx) + + with open(a_c) as f: + a_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + + def test_recythonize_py_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_py = os.path.join(src_dir, 'a.py') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_py, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.py" are "a.pxd" and "a.py". + self.assertEqual({a_pxd, a_py}, dep_tree.all_dependencies(a_py)) + + # Cythonize to create a.c + fresh_cythonize(a_py) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_py) + + with open(a_c) as f: + a_c_contents2 = f.read() + + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + def test_recythonize_pyx_on_dep_pxd_change(self): + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pyx = os.path.join(src_dir, 'b.pyx') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pyx, 'w') as f: + f.write('cimport a\n' + 'a.value = 2\n') + + + # The dependencies for "b.pyx" are "a.pxd" and "b.pyx". + self.assertEqual({a_pxd, b_pyx}, dep_tree.all_dependencies(b_pyx)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_pyx]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_pyx]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) + + + + def test_recythonize_py_on_dep_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pxd = os.path.join(src_dir, 'b.pxd') + b_py = os.path.join(src_dir, 'b.py') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pxd, 'w') as f: + f.write('cimport a\n') + + with open(b_py, 'w') as f: + f.write('a.value = 2\n') + + + # The dependencies for b.py are "a.pxd", "b.pxd" and "b.py". + self.assertEqual({a_pxd, b_pxd, b_py}, dep_tree.all_dependencies(b_py)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_py]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_py]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestStripLiterals.py cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestStripLiterals.py --- cython-0.20.1+1~201611251650-6686/Cython/Build/Tests/TestStripLiterals.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Build/Tests/TestStripLiterals.py 2022-03-24 10:16:46.000000000 +0000 @@ -6,10 +6,10 @@ def t(self, before, expected): actual, literals = strip_string_literals(before, prefix="_L") - self.assertEquals(expected, actual) + self.assertEqual(expected, actual) for key, value in literals.items(): actual = actual.replace(key, value) - self.assertEquals(before, actual) + self.assertEqual(before, actual) def test_empty(self): self.t("", "") @@ -54,4 +54,3 @@ def test_extern(self): self.t("cdef extern from 'a.h': # comment", "cdef extern from '_L1_': #_L2_") - diff -Nru cython-0.20.1+1~201611251650-6686/Cython/CodeWriter.py cython-0.20.1+1~202203241016-9537/Cython/CodeWriter.py --- cython-0.20.1+1~201611251650-6686/Cython/CodeWriter.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/CodeWriter.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,6 @@ """ Serializes a Cython code tree to Cython code. This is primarily useful for debugging and testing purposes. - The output is in a strict format, no whitespace or comments from the input is preserved (and it could not be as it is not present in the code tree). """ @@ -10,6 +9,7 @@ from .Compiler.Visitor import TreeVisitor from .Compiler.ExprNodes import * +from .Compiler.Nodes import CSimpleBaseTypeNode class LinesResult(object): @@ -28,7 +28,11 @@ self.put(s) self.newline() + class DeclarationWriter(TreeVisitor): + """ + A Cython code writer that is limited to declarations nodes. + """ indent_string = u" " @@ -76,6 +80,14 @@ self.visit(item.default) self.put(u", ") self.visit(items[-1]) + if output_rhs and items[-1].default is not None: + self.put(u" = ") + self.visit(items[-1].default) + + def _visit_indented(self, node): + self.indent() + self.visit(node) + self.dedent() def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) @@ -85,16 +97,14 @@ def visit_StatListNode(self, node): self.visitchildren(node) - + def visit_CDefExternNode(self, node): if node.include_file is None: file = u'*' else: file = u'"%s"' % node.include_file self.putline(u"cdef extern from %s:" % file) - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CPtrDeclaratorNode(self, node): self.put('*') @@ -111,13 +121,6 @@ self.visit(node.dimension) self.put(u']') - def visit_CArrayDeclaratorNode(self, node): - self.visit(node.base) - self.put(u'[') - if node.dimension is not None: - self.visit(node.dimension) - self.put(u']') - def visit_CFuncDeclaratorNode(self, node): # TODO: except, gil, etc. self.visit(node.base) @@ -136,13 +139,12 @@ self.put("short " * -node.longness) elif node.longness > 0: self.put("long " * node.longness) - self.put(node.name) + if node.name is not None: + self.put(node.name) def visit_CComplexBaseTypeNode(self, node): - self.put(u'(') self.visit(node.base_type) self.visit(node.declarator) - self.put(u')') def visit_CNestedBaseTypeNode(self, node): self.visit(node.base_type) @@ -162,7 +164,7 @@ self.comma_separated_list(node.declarators, output_rhs=True) self.endline() - def visit_container_node(self, node, decl, extras, attributes): + def _visit_container_node(self, node, decl, extras, attributes): # TODO: visibility self.startline(decl) if node.name: @@ -191,7 +193,7 @@ if node.packed: decl += u'packed ' decl += node.kind - self.visit_container_node(node, decl, None, node.attributes) + self._visit_container_node(node, decl, None, node.attributes) def visit_CppClassNode(self, node): extras = "" @@ -199,10 +201,10 @@ extras = u"[%s]" % ", ".join(node.templates) if node.base_classes: extras += "(%s)" % ", ".join(node.base_classes) - self.visit_container_node(node, u"cdef cppclass", extras, node.attributes) + self._visit_container_node(node, u"cdef cppclass", extras, node.attributes) def visit_CEnumDefNode(self, node): - self.visit_container_node(node, u"cdef enum", None, node.items) + self._visit_container_node(node, u"cdef enum", None, node.items) def visit_CEnumDefItemNode(self, node): self.startline(node.name) @@ -228,9 +230,7 @@ self.put(node.base_class_name) self.put(u")") self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CTypeDefNode(self, node): self.startline(u"ctypedef ") @@ -240,17 +240,49 @@ self.endline() def visit_FuncDefNode(self, node): + # TODO: support cdef + cpdef functions self.startline(u"def %s(" % node.name) self.comma_separated_list(node.args) self.endline(u"):") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) + + def visit_CFuncDefNode(self, node): + self.startline(u'cpdef ' if node.overridable else u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + + if node.base_type: + self.visit(node.base_type) + if node.base_type.name is not None: + self.put(u' ') + + # visit the CFuncDeclaratorNode, but put a `:` at the end of line + self.visit(node.declarator.base) + self.put(u'(') + self.comma_separated_list(node.declarator.args) + self.endline(u'):') + + self._visit_indented(node.body) def visit_CArgDeclNode(self, node): - if node.base_type.name is not None: + # For "CSimpleBaseTypeNode", the variable type may have been parsed as type. + # For other node types, the "name" is always None. + if not isinstance(node.base_type, CSimpleBaseTypeNode) or \ + node.base_type.name is not None: self.visit(node.base_type) - self.put(u" ") + + # If we printed something for "node.base_type", we may need to print an extra ' '. + # + # Special case: if "node.declarator" is a "CNameDeclaratorNode", + # its "name" might be an empty string, for example, for "cdef f(x)". + if node.declarator.declared_name(): + self.put(u" ") self.visit(node.declarator) if node.default is not None: self.put(u" = ") @@ -284,46 +316,20 @@ def visit_NameNode(self, node): self.put(node.name) - def visit_IntNode(self, node): - self.put(node.value) - - def visit_NoneNode(self, node): - self.put(u"None") - - def visit_NotNode(self, node): - self.put(u"(not ") - self.visit(node.operand) - self.put(u")") - def visit_DecoratorNode(self, node): self.startline("@") self.visit(node.decorator) self.endline() - def visit_BinopNode(self, node): - self.visit(node.operand1) - self.put(u" %s " % node.operator) - self.visit(node.operand2) - - def visit_AttributeNode(self, node): - self.visit(node.obj) - self.put(u".%s" % node.attribute) - - def visit_BoolNode(self, node): - self.put(str(node.value)) - - # FIXME: represent string nodes correctly - def visit_StringNode(self, node): - value = node.value - if value.encoding is not None: - value = value.encode(value.encoding) - self.put(repr(value)) - def visit_PassStatNode(self, node): self.startline(u"pass") self.endline() -class CodeWriter(DeclarationWriter): + +class StatementWriter(DeclarationWriter): + """ + A Cython code writer for most language statement features. + """ def visit_SingleAssignmentNode(self, node): self.startline() @@ -349,69 +355,51 @@ def visit_ForInStatNode(self, node): self.startline(u"for ") - self.visit(node.target) + if node.target.is_sequence_constructor: + self.comma_separated_list(node.target.args) + else: + self.visit(node.target) self.put(u" in ") self.visit(node.iterator.sequence) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) if node.else_clause is not None: self.line(u"else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) def visit_IfStatNode(self, node): - # The IfClauseNode is handled directly without a seperate match + # The IfClauseNode is handled directly without a separate match # for clariy. self.startline(u"if ") self.visit(node.if_clauses[0].condition) self.endline(":") - self.indent() - self.visit(node.if_clauses[0].body) - self.dedent() + self._visit_indented(node.if_clauses[0].body) for clause in node.if_clauses[1:]: self.startline("elif ") self.visit(clause.condition) self.endline(":") - self.indent() - self.visit(clause.body) - self.dedent() + self._visit_indented(clause.body) if node.else_clause is not None: self.line("else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) - def visit_SequenceNode(self, node): - self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... + def visit_WhileStatNode(self, node): + self.startline(u"while ") + self.visit(node.condition) + self.endline(u":") + self._visit_indented(node.body) + if node.else_clause is not None: + self.line("else:") + self._visit_indented(node.else_clause) - def visit_SimpleCallNode(self, node): - self.visit(node.function) - self.put(u"(") - self.comma_separated_list(node.args) - self.put(")") + def visit_ContinueStatNode(self, node): + self.line(u"continue") - def visit_GeneralCallNode(self, node): - self.visit(node.function) - self.put(u"(") - posarg = node.positional_args - if isinstance(posarg, AsTupleNode): - self.visit(posarg.arg) - else: - self.comma_separated_list(posarg.args) # TupleNode.args - if node.keyword_args: - if isinstance(node.keyword_args, DictNode): - for i, (name, value) in enumerate(node.keyword_args.key_value_pairs): - if i > 0: - self.put(', ') - self.visit(name) - self.put('=') - self.visit(value) - else: - raise Exception("Not implemented yet") - self.put(u")") + def visit_BreakStatNode(self, node): + self.line(u"break") + + def visit_SequenceNode(self, node): + self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... def visit_ExprStatNode(self, node): self.startline() @@ -433,25 +421,17 @@ self.put(u" as ") self.visit(node.target) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_TryFinallyStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) self.line(u"finally:") - self.indent() - self.visit(node.finally_clause) - self.dedent() + self._visit_indented(node.finally_clause) def visit_TryExceptStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) for x in node.except_clauses: self.visit(x) if node.else_clause is not None: @@ -466,13 +446,13 @@ self.put(u", ") self.visit(node.target) self.endline(":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_ReturnStatNode(self, node): - self.startline("return ") - self.visit(node.value) + self.startline("return") + if node.value is not None: + self.put(u" ") + self.visit(node.value) self.endline() def visit_ReraiseStatNode(self, node): @@ -498,24 +478,340 @@ self.put(self.tempnames[node.handle]) -class PxdWriter(DeclarationWriter): +class ExpressionWriter(TreeVisitor): + """ + A Cython code writer that is intentionally limited to expressions. + """ + + def __init__(self, result=None): + super(ExpressionWriter, self).__init__() + if result is None: + result = u"" + self.result = result + self.precedence = [0] + + def write(self, tree): + self.visit(tree) + return self.result + + def put(self, s): + self.result += s + + def remove(self, s): + if self.result.endswith(s): + self.result = self.result[:-len(s)] + + def comma_separated_list(self, items): + if len(items) > 0: + for item in items[:-1]: + self.visit(item) + self.put(u", ") + self.visit(items[-1]) + + def visit_Node(self, node): + raise AssertionError("Node not handled by serializer: %r" % node) + + def visit_IntNode(self, node): + self.put(node.value) + + def visit_FloatNode(self, node): + self.put(node.value) + + def visit_NoneNode(self, node): + self.put(u"None") + + def visit_NameNode(self, node): + self.put(node.name) + + def visit_EllipsisNode(self, node): + self.put(u"...") + + def visit_BoolNode(self, node): + self.put(str(node.value)) + + def visit_ConstNode(self, node): + self.put(str(node.value)) + + def visit_ImagNode(self, node): + self.put(node.value) + self.put(u"j") + + def emit_string(self, node, prefix=u""): + repr_val = repr(node.value) + if repr_val[0] in 'ub': + repr_val = repr_val[1:] + self.put(u"%s%s" % (prefix, repr_val)) + + def visit_BytesNode(self, node): + self.emit_string(node, u"b") + + def visit_StringNode(self, node): + self.emit_string(node) + + def visit_UnicodeNode(self, node): + self.emit_string(node, u"u") + + def emit_sequence(self, node, parens=(u"", u"")): + open_paren, close_paren = parens + items = node.subexpr_nodes() + self.put(open_paren) + self.comma_separated_list(items) + self.put(close_paren) + + def visit_ListNode(self, node): + self.emit_sequence(node, u"[]") + + def visit_TupleNode(self, node): + self.emit_sequence(node, u"()") + + def visit_SetNode(self, node): + if len(node.subexpr_nodes()) > 0: + self.emit_sequence(node, u"{}") + else: + self.put(u"set()") + + def visit_DictNode(self, node): + self.emit_sequence(node, u"{}") + + def visit_DictItemNode(self, node): + self.visit(node.key) + self.put(u": ") + self.visit(node.value) + + unop_precedence = { + 'not': 3, '!': 3, + '+': 11, '-': 11, '~': 11, + } + binop_precedence = { + 'or': 1, + 'and': 2, + # unary: 'not': 3, '!': 3, + 'in': 4, 'not_in': 4, 'is': 4, 'is_not': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, '!=': 4, '==': 4, + '|': 5, + '^': 6, + '&': 7, + '<<': 8, '>>': 8, + '+': 9, '-': 9, + '*': 10, '@': 10, '/': 10, '//': 10, '%': 10, + # unary: '+': 11, '-': 11, '~': 11 + '**': 12, + } + + def operator_enter(self, new_prec): + old_prec = self.precedence[-1] + if old_prec > new_prec: + self.put(u"(") + self.precedence.append(new_prec) + + def operator_exit(self): + old_prec, new_prec = self.precedence[-2:] + if old_prec > new_prec: + self.put(u")") + self.precedence.pop() + + def visit_NotNode(self, node): + op = 'not' + prec = self.unop_precedence[op] + self.operator_enter(prec) + self.put(u"not ") + self.visit(node.operand) + self.operator_exit() + + def visit_UnopNode(self, node): + op = node.operator + prec = self.unop_precedence[op] + self.operator_enter(prec) + self.put(u"%s" % node.operator) + self.visit(node.operand) + self.operator_exit() + + def visit_BinopNode(self, node): + op = node.operator + prec = self.binop_precedence.get(op, 0) + self.operator_enter(prec) + self.visit(node.operand1) + self.put(u" %s " % op.replace('_', ' ')) + self.visit(node.operand2) + self.operator_exit() + + def visit_BoolBinopNode(self, node): + self.visit_BinopNode(node) + + def visit_PrimaryCmpNode(self, node): + self.visit_BinopNode(node) + + def visit_IndexNode(self, node): + self.visit(node.base) + self.put(u"[") + if isinstance(node.index, TupleNode): + self.emit_sequence(node.index) + else: + self.visit(node.index) + self.put(u"]") + + def visit_SliceIndexNode(self, node): + self.visit(node.base) + self.put(u"[") + if node.start: + self.visit(node.start) + self.put(u":") + if node.stop: + self.visit(node.stop) + if node.slice: + self.put(u":") + self.visit(node.slice) + self.put(u"]") + + def visit_SliceNode(self, node): + if not node.start.is_none: + self.visit(node.start) + self.put(u":") + if not node.stop.is_none: + self.visit(node.stop) + if not node.step.is_none: + self.put(u":") + self.visit(node.step) + + def visit_CondExprNode(self, node): + self.visit(node.true_val) + self.put(u" if ") + self.visit(node.test) + self.put(u" else ") + self.visit(node.false_val) + + def visit_AttributeNode(self, node): + self.visit(node.obj) + self.put(u".%s" % node.attribute) + + def visit_SimpleCallNode(self, node): + self.visit(node.function) + self.put(u"(") + self.comma_separated_list(node.args) + self.put(")") + + def emit_pos_args(self, node): + if node is None: + return + if isinstance(node, AddNode): + self.emit_pos_args(node.operand1) + self.emit_pos_args(node.operand2) + elif isinstance(node, TupleNode): + for expr in node.subexpr_nodes(): + self.visit(expr) + self.put(u", ") + elif isinstance(node, AsTupleNode): + self.put("*") + self.visit(node.arg) + self.put(u", ") + else: + self.visit(node) + self.put(u", ") + + def emit_kwd_args(self, node): + if node is None: + return + if isinstance(node, MergedDictNode): + for expr in node.subexpr_nodes(): + self.emit_kwd_args(expr) + elif isinstance(node, DictNode): + for expr in node.subexpr_nodes(): + self.put(u"%s=" % expr.key.value) + self.visit(expr.value) + self.put(u", ") + else: + self.put(u"**") + self.visit(node) + self.put(u", ") + + def visit_GeneralCallNode(self, node): + self.visit(node.function) + self.put(u"(") + self.emit_pos_args(node.positional_args) + self.emit_kwd_args(node.keyword_args) + self.remove(u", ") + self.put(")") + + def emit_comprehension(self, body, target, + sequence, condition, + parens=(u"", u"")): + open_paren, close_paren = parens + self.put(open_paren) + self.visit(body) + self.put(u" for ") + self.visit(target) + self.put(u" in ") + self.visit(sequence) + if condition: + self.put(u" if ") + self.visit(condition) + self.put(close_paren) + + def visit_ComprehensionAppendNode(self, node): + self.visit(node.expr) + + def visit_DictComprehensionAppendNode(self, node): + self.visit(node.key_expr) + self.put(u": ") + self.visit(node.value_expr) + + def visit_ComprehensionNode(self, node): + tpmap = {'list': u"[]", 'dict': u"{}", 'set': u"{}"} + parens = tpmap[node.type.py_type_name()] + body = node.loop.body + target = node.loop.target + sequence = node.loop.iterator.sequence + condition = None + if hasattr(body, 'if_clauses'): + # type(body) is Nodes.IfStatNode + condition = body.if_clauses[0].condition + body = body.if_clauses[0].body + self.emit_comprehension(body, target, sequence, condition, parens) + + def visit_GeneratorExpressionNode(self, node): + body = node.loop.body + target = node.loop.target + sequence = node.loop.iterator.sequence + condition = None + if hasattr(body, 'if_clauses'): + # type(body) is Nodes.IfStatNode + condition = body.if_clauses[0].condition + body = body.if_clauses[0].body.expr.arg + elif hasattr(body, 'expr'): + # type(body) is Nodes.ExprStatNode + body = body.expr.arg + self.emit_comprehension(body, target, sequence, condition, u"()") + + +class PxdWriter(DeclarationWriter, ExpressionWriter): + """ + A Cython code writer for everything supported in pxd files. + (currently unused) + """ + def __call__(self, node): print(u'\n'.join(self.write(node).lines)) return node def visit_CFuncDefNode(self, node): - if 'inline' in node.modifiers: - return if node.overridable: self.startline(u'cpdef ') else: self.startline(u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') if node.visibility != 'private': self.put(node.visibility) self.put(u' ') if node.api: self.put(u'api ') self.visit(node.declarator) - + def visit_StatNode(self, node): pass + + +class CodeWriter(StatementWriter, ExpressionWriter): + """ + A complete Cython code writer. + """ diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/AnalysedTreeTransforms.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/AnalysedTreeTransforms.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/AnalysedTreeTransforms.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/AnalysedTreeTransforms.py 2022-03-24 10:16:46.000000000 +0000 @@ -10,9 +10,9 @@ class AutoTestDictTransform(ScopeTrackingTransform): # Handles autotestdict directive - blacklist = ['__cinit__', '__dealloc__', '__richcmp__', - '__nonzero__', '__bool__', - '__len__', '__contains__'] + excludelist = ['__cinit__', '__dealloc__', '__richcmp__', + '__nonzero__', '__bool__', + '__len__', '__contains__'] def visit_ModuleNode(self, node): if node.is_pxd: @@ -81,7 +81,7 @@ name = node.entry.name else: name = node.name - if self.scope_type == 'cclass' and name in self.blacklist: + if self.scope_type == 'cclass' and name in self.excludelist: return node if self.scope_type == 'pyclass': class_name = self.scope_node.name diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Annotate.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Annotate.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Annotate.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Annotate.py 2022-03-24 10:16:46.000000000 +0000 @@ -23,8 +23,12 @@ class AnnotationCCodeWriter(CCodeWriter): - def __init__(self, create_from=None, buffer=None, copy_formatting=True): + # also used as marker for detection of complete code emission in tests + COMPLETE_CODE_TITLE = "Complete cythonized code" + + def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False, source_desc=None): CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting) + self.show_entire_c_code = show_entire_c_code if create_from is None: self.annotation_buffer = StringIO() self.last_annotated_pos = None @@ -79,19 +83,11 @@ css.append(HtmlFormatter().get_style_defs('.cython')) return '\n'.join(css) - _js = """ - function toggleDiv(id) { - theDiv = id.nextElementSibling - if (theDiv.style.display != 'block') theDiv.style.display = 'block'; - else theDiv.style.display = 'none'; - } - """.strip() - _css_template = textwrap.dedent(""" body.cython { font-family: courier; font-size: 12; } .cython.tag { } - .cython.line { margin: 0em } + .cython.line { color: #000000; margin: 0em } .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; } .cython.line .run { background-color: #B0FFB0; } @@ -114,6 +110,14 @@ .cython.code .c_call { color: #0000FF; } """) + # on-click toggle function to show/hide C source code + _onclick_attr = ' onclick="{0}"'.format(( + "(function(s){" + " s.display = s.display === 'block' ? 'none' : 'block'" + "})(this.nextElementSibling.style)" + ).replace(' ', '') # poor dev's JS minification + ) + def save_annotation(self, source_filename, target_filename, coverage_xml=None): with Utils.open_source_file(source_filename) as f: code = f.read() @@ -141,9 +145,6 @@ -

Generated by Cython {watermark}{more_info}

@@ -151,7 +152,7 @@ Yellow lines hint at Python interaction.
Click on a line that starts with a "+" to see the C code that Cython generated for it.

- ''').format(css=self._css(), js=self._js, watermark=Version.watermark, + ''').format(css=self._css(), watermark=Version.watermark, filename=os.path.basename(source_filename) if source_filename else '', more_info=coverage_info) ] @@ -201,17 +202,24 @@ for line in coverage_data.iterfind('lines/line') ) - def _htmlify_code(self, code): + def _htmlify_code(self, code, language): try: from pygments import highlight - from pygments.lexers import CythonLexer + from pygments.lexers import CythonLexer, CppLexer from pygments.formatters import HtmlFormatter except ImportError: # no Pygments, just escape the code return html_escape(code) + if language == "cython": + lexer = CythonLexer(stripnl=False, stripall=False) + elif language == "c/cpp": + lexer = CppLexer(stripnl=False, stripall=False) + else: + # unknown language, use fallback + return html_escape(code) html_code = highlight( - code, CythonLexer(stripnl=False, stripall=False), + code, lexer, HtmlFormatter(nowrap=True)) return html_code @@ -231,7 +239,7 @@ return u"%s" % ( group_name, match.group(group_name)) - lines = self._htmlify_code(cython_code).splitlines() + lines = self._htmlify_code(cython_code, "cython").splitlines() lineno_width = len(str(len(lines))) if not covered_lines: covered_lines = None @@ -253,7 +261,7 @@ calls['py_macro_api'] + calls['pyx_macro_api']) if c_code: - onclick = " onclick='toggleDiv(this)'" + onclick = self._onclick_attr expandsymbol = '+' else: onclick = '' @@ -282,6 +290,19 @@ outlist.append(u"
{code}
".format( score=score, covered=covered, code=c_code)) outlist.append(u"") + + # now the whole c-code if needed: + if self.show_entire_c_code: + outlist.append(u'

') + onclick_title = u"
+ {title}
\n" + outlist.append(onclick_title.format( + onclick=self._onclick_attr, + title=AnnotationCCodeWriter.COMPLETE_CODE_TITLE, + )) + complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp") + outlist.append(u"
{code}
".format(code=complete_code_as_html)) + outlist.append(u"

") + return outlist @@ -294,7 +315,7 @@ br'(?PPy[A-Z][a-z]+_[A-Z][A-Z_]+)|' br'(?PPy[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]*)' br')(?=\()|' # look-ahead to exclude subsequent '(' from replacement - br'(?P(?:(?<=;) *if .* +)?\{__pyx_filename = .*goto __pyx_L\w+;\})' + br'(?P(?:(?<=;) *if [^;]* +)?__PYX_ERR\([^)]+\))' ).decode('ascii')).sub diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/AutoDocTransforms.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/AutoDocTransforms.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/AutoDocTransforms.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/AutoDocTransforms.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,91 +1,97 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function from .Visitor import CythonTransform from .StringEncoding import EncodedString from . import Options -from . import PyrexTypes, ExprNodes +from . import PyrexTypes +from ..CodeWriter import ExpressionWriter +from .Errors import warning + + +class AnnotationWriter(ExpressionWriter): + """ + A Cython code writer for Python expressions in argument/variable annotations. + """ + def __init__(self, description=None): + """description is optional. If specified it is used in + warning messages for the nodes that don't convert to string properly. + If not specified then no messages are generated. + """ + ExpressionWriter.__init__(self) + self.description = description + self.incomplete = False + + def visit_Node(self, node): + self.put(u"") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert code to string representation in {0}".format( + self.description), level=1) + + def visit_LambdaNode(self, node): + # XXX Should we do better? + self.put("") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert lambda to string representation in {0}".format( + self.description), level=1) + + def visit_UnicodeNode(self, node): + # Discard Unicode prefix in annotations. Any tool looking at them + # would probably expect Py3 string semantics. + self.emit_string(node, "") + + def visit_AnnotationNode(self, node): + self.put(node.string.unicode_value) + class EmbedSignature(CythonTransform): def __init__(self, context): super(EmbedSignature, self).__init__(context) - self.denv = None # XXX self.class_name = None self.class_node = None - unop_precedence = 11 - binop_precedence = { - 'or': 1, - 'and': 2, - 'not': 3, - 'in': 4, 'not in': 4, 'is': 4, 'is not': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, '!=': 4, '==': 4, - '|': 5, - '^': 6, - '&': 7, - '<<': 8, '>>': 8, - '+': 9, '-': 9, - '*': 10, '/': 10, '//': 10, '%': 10, - # unary: '+': 11, '-': 11, '~': 11 - '**': 12} - - def _fmt_expr_node(self, node, precedence=0): - if isinstance(node, ExprNodes.BinopNode) and not node.inplace: - new_prec = self.binop_precedence.get(node.operator, 0) - result = '%s %s %s' % (self._fmt_expr_node(node.operand1, new_prec), - node.operator, - self._fmt_expr_node(node.operand2, new_prec)) - if precedence > new_prec: - result = '(%s)' % result - elif isinstance(node, ExprNodes.UnopNode): - result = '%s%s' % (node.operator, - self._fmt_expr_node(node.operand, self.unop_precedence)) - if precedence > self.unop_precedence: - result = '(%s)' % result - elif isinstance(node, ExprNodes.AttributeNode): - result = '%s.%s' % (self._fmt_expr_node(node.obj), node.attribute) - else: - result = node.name + def _fmt_expr(self, node): + writer = ExpressionWriter() + result = writer.write(node) + # print(type(node).__name__, '-->', result) return result - def _fmt_arg_defv(self, arg): - default_val = arg.default - if not default_val: - return None - if isinstance(default_val, ExprNodes.NullNode): - return 'NULL' - try: - denv = self.denv # XXX - ctval = default_val.compile_time_value(self.denv) - repr_val = repr(ctval) - if isinstance(default_val, ExprNodes.UnicodeNode): - if repr_val[:1] != 'u': - return u'u%s' % repr_val - elif isinstance(default_val, ExprNodes.BytesNode): - if repr_val[:1] != 'b': - return u'b%s' % repr_val - elif isinstance(default_val, ExprNodes.StringNode): - if repr_val[:1] in 'ub': - return repr_val[1:] - return repr_val - except Exception: - try: - return self._fmt_expr_node(default_val) - except AttributeError: - return '' + def _fmt_annotation(self, node): + writer = AnnotationWriter() + result = writer.write(node) + # print(type(node).__name__, '-->', result) + return result def _fmt_arg(self, arg): if arg.type is PyrexTypes.py_object_type or arg.is_self_arg: doc = arg.name else: doc = arg.type.declaration_code(arg.name, for_display=1) - if arg.default: - arg_defv = self._fmt_arg_defv(arg) - if arg_defv: - doc = doc + ('=%s' % arg_defv) + + if arg.annotation: + annotation = self._fmt_annotation(arg.annotation) + doc = doc + (': %s' % annotation) + if arg.default: + default = self._fmt_expr(arg.default) + doc = doc + (' = %s' % default) + elif arg.default: + default = self._fmt_expr(arg.default) + doc = doc + ('=%s' % default) return doc + def _fmt_star_arg(self, arg): + arg_doc = arg.name + if arg.annotation: + annotation = self._fmt_annotation(arg.annotation) + arg_doc = arg_doc + (': %s' % annotation) + return arg_doc + def _fmt_arglist(self, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, hide_self=False): arglist = [] @@ -94,11 +100,15 @@ arg_doc = self._fmt_arg(arg) arglist.append(arg_doc) if pargs: - arglist.insert(npargs, '*%s' % pargs.name) + arg_doc = self._fmt_star_arg(pargs) + arglist.insert(npargs + npoargs, '*%s' % arg_doc) elif nkargs: - arglist.insert(npargs, '*') + arglist.insert(npargs + npoargs, '*') + if npoargs: + arglist.insert(npoargs, '/') if kargs: - arglist.append('**%s' % kargs.name) + arg_doc = self._fmt_star_arg(kargs) + arglist.append('**%s' % arg_doc) return arglist def _fmt_ret_type(self, ret): @@ -108,21 +118,25 @@ return ret.declaration_code("", for_display=1) def _fmt_signature(self, cls_name, func_name, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, + return_expr=None, return_type=None, hide_self=False): arglist = self._fmt_arglist(args, - npargs, pargs, + npoargs, npargs, pargs, nkargs, kargs, hide_self=hide_self) arglist_doc = ', '.join(arglist) func_doc = '%s(%s)' % (func_name, arglist_doc) if cls_name: func_doc = '%s.%s' % (cls_name, func_doc) - if return_type: + ret_doc = None + if return_expr: + ret_doc = self._fmt_annotation(return_expr) + elif return_type: ret_doc = self._fmt_ret_type(return_type) - if ret_doc: - func_doc = '%s -> %s' % (func_doc, ret_doc) + if ret_doc: + func_doc = '%s -> %s' % (func_doc, ret_doc) return func_doc def _embed_signature(self, signature, node_doc): @@ -171,12 +185,14 @@ else: class_name, func_name = self.class_name, node.name + npoargs = getattr(node, 'num_posonly_args', 0) nkargs = getattr(node, 'num_kwonly_args', 0) - npargs = len(node.args) - nkargs + npargs = len(node.args) - nkargs - npoargs signature = self._fmt_signature( class_name, func_name, node.args, - npargs, node.star_arg, + npoargs, npargs, node.star_arg, nkargs, node.starstar_arg, + return_expr=node.return_type_annotation, return_type=None, hide_self=hide_self) if signature: if is_constructor: @@ -199,7 +215,7 @@ def visit_CFuncDefNode(self, node): if not self.current_directives['embedsignature']: return node - if not node.overridable: # not cpdef FOO(...): + if not node.overridable: # not cpdef FOO(...): return node signature = self._fmt_signature( @@ -215,8 +231,9 @@ old_doc = None new_doc = self._embed_signature(signature, old_doc) node.entry.doc = EncodedString(new_doc) - if hasattr(node, 'py_func') and node.py_func is not None: - node.py_func.entry.doc = EncodedString(new_doc) + py_func = getattr(node, 'py_func', None) + if py_func is not None: + py_func.entry.doc = EncodedString(new_doc) return node def visit_PropertyNode(self, node): diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Buffer.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Buffer.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Buffer.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Buffer.py 2022-03-24 10:16:46.000000000 +0000 @@ -12,7 +12,6 @@ from . import Naming from . import Symtab - def dedent(text, reindent=0): from textwrap import dedent text = dedent(text) @@ -37,7 +36,6 @@ if self.buffers_exists: use_bufstruct_declare_code(node.scope) use_py2_buffer_functions(node.scope) - node.scope.use_utility_code(empty_bufstruct_utility) return result @@ -87,7 +85,7 @@ aux_var = scope.declare_var(name=None, cname=cname, type=type, pos=node.pos) if entry.is_arg: - aux_var.used = True # otherwise, NameNode will mark whether it is used + aux_var.used = True # otherwise, NameNode will mark whether it is used return aux_var @@ -113,9 +111,9 @@ # # Analysis # -buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! +buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! buffer_defaults = {"ndim": 1, "mode": "full", "negative_indices": True, "cast": False} -buffer_positional_options_count = 1 # anything beyond this needs keyword argument +buffer_positional_options_count = 1 # anything beyond this needs keyword argument ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option' ERR_BUF_TOO_MANY = 'Too many buffer options' @@ -148,12 +146,12 @@ options = {} for name, (value, pos) in dictargs.items(): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) options[name] = value for name, (value, pos) in zip(buffer_options, posargs): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) if name in options: raise CompileError(pos, ERR_BUF_DUP % name) @@ -161,7 +159,7 @@ # Check that they are all there and copy defaults for name in buffer_options: - if not name in options: + if name not in options: try: options[name] = defaults[name] except KeyError: @@ -300,9 +298,10 @@ ln = [] for i in range(buf_entry.type.ndim): for fldname in fldnames: - ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % \ - (pybuffernd_struct, i, fldname, - pybuffernd_struct, fldname, i)) + ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % ( + pybuffernd_struct, i, fldname, + pybuffernd_struct, fldname, i, + )) code.putln(' '.join(ln)) def put_init_vars(entry, code): @@ -318,8 +317,8 @@ code.putln("%s.data = NULL;" % pybuffernd_struct) code.putln("%s.rcbuffer = &%s;" % (pybuffernd_struct, pybuffer_struct)) + def put_acquire_arg_buffer(entry, code, pos): - code.globalstate.use_utility_code(acquire_utility_code) buffer_aux = entry.buffer_aux getbuffer = get_getbuffer_call(code, entry.cname, buffer_aux, entry.type) @@ -328,14 +327,16 @@ code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % entry.type.dtype.struct_nesting_depth()) code.putln(code.error_goto_if("%s == -1" % getbuffer, pos)) code.putln("}") - # An exception raised in arg parsing cannot be catched, so no + # An exception raised in arg parsing cannot be caught, so no # need to care about the buffer then. put_unpack_buffer_aux_into_scope(entry, code) + def put_release_buffer_code(code, entry): code.globalstate.use_utility_code(acquire_utility_code) code.putln("__Pyx_SafeReleaseBuffer(&%s.rcbuffer->pybuffer);" % entry.buffer_aux.buflocal_nd_var.cname) + def get_getbuffer_call(code, obj_cname, buffer_aux, buffer_type): ndim = buffer_type.ndim cast = int(buffer_type.cast) @@ -344,10 +345,12 @@ dtype_typeinfo = get_type_information_cname(code, buffer_type.dtype) + code.globalstate.use_utility_code(acquire_utility_code) return ("__Pyx_GetBufferAndValidate(&%(pybuffernd_struct)s.rcbuffer->pybuffer, " "(PyObject*)%(obj_cname)s, &%(dtype_typeinfo)s, %(flags)s, %(ndim)d, " "%(cast)d, __pyx_stack)" % locals()) + def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, is_initialized, pos, code): """ @@ -365,14 +368,13 @@ """ buffer_aux, buffer_type = buf_entry.buffer_aux, buf_entry.type - code.globalstate.use_utility_code(acquire_utility_code) pybuffernd_struct = buffer_aux.buflocal_nd_var.cname flags = get_flags(buffer_aux, buffer_type) - code.putln("{") # Set up necesarry stack for getbuffer + code.putln("{") # Set up necessary stack for getbuffer code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth()) - getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below + getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below if is_initialized: # Release any existing buffer @@ -385,18 +387,19 @@ # before raising the exception. A failure of reacquisition # will cause the reacquisition exception to be reported, one # can consider working around this later. - type, value, tb = [code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=False) - for i in range(3)] - code.putln('PyErr_Fetch(&%s, &%s, &%s);' % (type, value, tb)) + exc_temps = tuple(code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=False) + for _ in range(3)) + code.putln('PyErr_Fetch(&%s, &%s, &%s);' % exc_temps) code.putln('if (%s) {' % code.unlikely("%s == -1" % (getbuffer % lhs_cname))) - code.putln('Py_XDECREF(%s); Py_XDECREF(%s); Py_XDECREF(%s);' % (type, value, tb)) # Do not refnanny these! + code.putln('Py_XDECREF(%s); Py_XDECREF(%s); Py_XDECREF(%s);' % exc_temps) # Do not refnanny these! code.globalstate.use_utility_code(raise_buffer_fallback_code) code.putln('__Pyx_RaiseBufferFallbackError();') code.putln('} else {') - code.putln('PyErr_Restore(%s, %s, %s);' % (type, value, tb)) - for t in (type, value, tb): - code.funcstate.release_temp(t) + code.putln('PyErr_Restore(%s, %s, %s);' % exc_temps) code.putln('}') + code.putln('%s = %s = %s = 0;' % exc_temps) + for t in exc_temps: + code.funcstate.release_temp(t) code.putln('}') # Unpack indices put_unpack_buffer_aux_into_scope(buf_entry, code) @@ -417,7 +420,7 @@ put_unpack_buffer_aux_into_scope(buf_entry, code) code.putln('}') - code.putln("}") # Release stack + code.putln("}") # Release stack def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, @@ -490,15 +493,6 @@ env.use_utility_code(buffer_struct_declare_code) -def get_empty_bufstruct_code(max_ndim): - code = dedent(""" - static Py_ssize_t __Pyx_zeros[] = {%s}; - static Py_ssize_t __Pyx_minusones[] = {%s}; - """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim)) - return UtilityCode(proto=code) - -empty_bufstruct_utility = get_empty_bufstruct_code(Options.buffer_max_dims) - def buf_lookup_full_code(proto, defin, name, nd): """ Generates a buffer lookup function for the right number @@ -519,6 +513,7 @@ """) % (i, i, i, i) for i in range(nd)] ) + "\nreturn ptr;\n}") + def buf_lookup_strided_code(proto, defin, name, nd): """ Generates a buffer lookup function for the right number @@ -529,6 +524,7 @@ offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)]) proto.putln("#define %s(type, buf, %s) (type)((char*)buf + %s)" % (name, args, offset)) + def buf_lookup_c_code(proto, defin, name, nd): """ Similar to strided lookup, but can assume that the last dimension @@ -542,6 +538,7 @@ offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd - 1)]) proto.putln("#define %s(type, buf, %s) ((type)((char*)buf + %s) + i%d)" % (name, args, offset, nd - 1)) + def buf_lookup_fortran_code(proto, defin, name, nd): """ Like C lookup, but the first index is optimized instead. @@ -557,6 +554,7 @@ def use_py2_buffer_functions(env): env.use_utility_code(GetAndReleaseBufferUtilityCode()) + class GetAndReleaseBufferUtilityCode(object): # Emulation of PyObject_GetBuffer and PyBuffer_Release for Python 2. # For >= 2.6 we do double mode -- use the new buffer interface on objects @@ -620,7 +618,7 @@ def mangle_dtype_name(dtype): - # Use prefixes to seperate user defined types from builtins + # Use prefixes to separate user defined types from builtins # (consider "typedef float unsigned_int") if dtype.is_pyobject: return "object" @@ -639,7 +637,7 @@ and return the name of the type info struct. Structs with two floats of the same size are encoded as complex numbers. - One can seperate between complex numbers declared as struct or with native + One can separate between complex numbers declared as struct or with native encoding by inspecting to see if the fields field of the type is filled in. """ @@ -671,16 +669,26 @@ if dtype.is_simple_buffer_dtype(): structinfo_name = "NULL" elif dtype.is_struct: - fields = dtype.scope.var_entries - # Must pre-call all used types in order not to recurse utility code - # writing. + struct_scope = dtype.scope + if dtype.is_cv_qualified: + struct_scope = struct_scope.base_type_scope + # Must pre-call all used types in order not to recurse during utility code writing. + fields = struct_scope.var_entries assert len(fields) > 0 types = [get_type_information_cname(code, f.type, maxdepth - 1) for f in fields] typecode.putln("static __Pyx_StructField %s[] = {" % structinfo_name, safe=True) + + if dtype.is_cv_qualified: + # roughly speaking, remove "const" from struct_type + struct_type = dtype.cv_base_type.empty_declaration_code() + else: + struct_type = dtype.empty_declaration_code() + for f, typeinfo in zip(fields, types): typecode.putln(' {&%s, "%s", offsetof(%s, %s)},' % - (typeinfo, f.name, dtype.empty_declaration_code(), f.cname), safe=True) + (typeinfo, f.name, struct_type, f.cname), safe=True) + typecode.putln(' {NULL, NULL, 0}', safe=True) typecode.putln("};", safe=True) else: @@ -724,25 +732,18 @@ else: return TempitaUtilityCode.load(util_code_name, "Buffer.c", context=context, **kwargs) -context = dict(max_dims=str(Options.buffer_max_dims)) -buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare", - context=context) - +context = dict(max_dims=Options.buffer_max_dims) +buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare", context=context) +buffer_formats_declare_code = load_buffer_utility("BufferFormatStructs") # Utility function to set the right exception # The caller should immediately goto_error raise_indexerror_code = load_buffer_utility("BufferIndexError") raise_indexerror_nogil = load_buffer_utility("BufferIndexErrorNogil") - raise_buffer_fallback_code = load_buffer_utility("BufferFallbackError") -buffer_structs_code = load_buffer_utility( - "BufferFormatStructs", proto_block='utility_code_proto_before_types') -acquire_utility_code = load_buffer_utility("BufferFormatCheck", - context=context, - requires=[buffer_structs_code]) + +acquire_utility_code = load_buffer_utility("BufferGetAndValidate", context=context) +buffer_format_check_code = load_buffer_utility("BufferFormatCheck", context=context) # See utility code BufferFormatFromTypeInfo -_typeinfo_to_format_code = load_buffer_utility("TypeInfoToFormat", context={}, - requires=[buffer_structs_code]) -typeinfo_compare_code = load_buffer_utility("TypeInfoCompare", context={}, - requires=[buffer_structs_code]) +_typeinfo_to_format_code = load_buffer_utility("TypeInfoToFormat") diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Builtin.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Builtin.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Builtin.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Builtin.py 2022-03-24 10:16:46.000000000 +0000 @@ -4,11 +4,11 @@ from __future__ import absolute_import -from .Symtab import BuiltinScope, StructOrUnionScope +from .StringEncoding import EncodedString +from .Symtab import BuiltinScope, StructOrUnionScope, ModuleScope from .Code import UtilityCode from .TypeSlots import Signature from . import PyrexTypes -from . import Options # C-level implementations of builtin types, functions and methods @@ -21,6 +21,7 @@ globals_utility_code = UtilityCode.load("Globals", "Builtins.c") builtin_utility_code = { + 'StopAsyncIteration': UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"), } @@ -29,17 +30,19 @@ class _BuiltinOverride(object): def __init__(self, py_name, args, ret_type, cname, py_equiv="*", utility_code=None, sig=None, func_type=None, - is_strict_signature=False, builtin_return_type=None): + is_strict_signature=False, builtin_return_type=None, + nogil=None): self.py_name, self.cname, self.py_equiv = py_name, cname, py_equiv self.args, self.ret_type = args, ret_type self.func_type, self.sig = func_type, sig self.builtin_return_type = builtin_return_type self.is_strict_signature = is_strict_signature self.utility_code = utility_code + self.nogil = nogil def build_func_type(self, sig=None, self_arg=None): if sig is None: - sig = Signature(self.args, self.ret_type) + sig = Signature(self.args, self.ret_type, nogil=self.nogil) sig.exception_check = False # not needed for the current builtins func_type = sig.function_type(self_arg) if self.is_strict_signature: @@ -53,7 +56,7 @@ def __init__(self, py_name, cname=None, field_type=None, field_type_name=None): self.py_name = py_name self.cname = cname or py_name - self.field_type_name = field_type_name # can't do the lookup before the type is declared! + self.field_type_name = field_type_name # can't do the lookup before the type is declared! self.field_type = field_type def declare_in_type(self, self_type): @@ -91,31 +94,40 @@ builtin_function_table = [ # name, args, return, C API func, py equiv = "*" BuiltinFunction('abs', "d", "d", "fabs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "f", "f", "fabsf", - is_strict_signature = True), - BuiltinFunction('abs', None, None, "__Pyx_abs_int", - utility_code = UtilityCode.load("abs_int", "Builtins.c"), + is_strict_signature=True, nogil=True), + BuiltinFunction('abs', "i", "i", "abs", + is_strict_signature=True, nogil=True), + BuiltinFunction('abs', "l", "l", "labs", + is_strict_signature=True, nogil=True), + BuiltinFunction('abs', None, None, "__Pyx_abs_longlong", + utility_code = UtilityCode.load("abs_longlong", "Builtins.c"), + func_type = PyrexTypes.CFuncType( + PyrexTypes.c_longlong_type, [ + PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_longlong_type, None) + ], + is_strict_signature = True, nogil=True)), + ] + list( + BuiltinFunction('abs', None, None, "/*abs_{0}*/".format(t.specialization_name()), func_type = PyrexTypes.CFuncType( - PyrexTypes.c_uint_type, [ - PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_int_type, None) - ], - is_strict_signature = True)), - BuiltinFunction('abs', None, None, "__Pyx_abs_long", - utility_code = UtilityCode.load("abs_long", "Builtins.c"), + t, + [PyrexTypes.CFuncTypeArg("arg", t, None)], + is_strict_signature = True, nogil=True)) + for t in (PyrexTypes.c_uint_type, PyrexTypes.c_ulong_type, PyrexTypes.c_ulonglong_type) + ) + list( + BuiltinFunction('abs', None, None, "__Pyx_c_abs{0}".format(t.funcsuffix), func_type = PyrexTypes.CFuncType( - PyrexTypes.c_ulong_type, [ - PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_long_type, None) + t.real_type, [ + PyrexTypes.CFuncTypeArg("arg", t, None) ], - is_strict_signature = True)), - BuiltinFunction('abs', None, None, "__Pyx_abs_longlong", - utility_code = UtilityCode.load("abs_longlong", "Builtins.c"), - func_type = PyrexTypes.CFuncType( - PyrexTypes.c_ulonglong_type, [ - PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_longlong_type, None) - ], - is_strict_signature = True)), - BuiltinFunction('abs', "O", "O", "PyNumber_Absolute"), + is_strict_signature = True, nogil=True)) + for t in (PyrexTypes.c_float_complex_type, + PyrexTypes.c_double_complex_type, + PyrexTypes.c_longdouble_complex_type) + ) + [ + BuiltinFunction('abs', "O", "O", "__Pyx_PyNumber_Absolute", + utility_code=UtilityCode.load("py_abs", "Builtins.c")), #('all', "", "", ""), #('any', "", "", ""), #('ascii', "", "", ""), @@ -143,7 +155,8 @@ utility_code=getattr3_utility_code), BuiltinFunction('getattr', "OO", "O", "__Pyx_GetAttr", utility_code=getattr_utility_code), - BuiltinFunction('hasattr', "OO", "b", "PyObject_HasAttr"), + BuiltinFunction('hasattr', "OO", "b", "__Pyx_HasAttr", + utility_code = UtilityCode.load("HasAttr", "Builtins.c")), BuiltinFunction('hash', "O", "h", "PyObject_Hash"), #('hex', "", "", ""), #('id', "", "", ""), @@ -192,13 +205,13 @@ #('raw_input', "", "", ""), #('reduce', "", "", ""), BuiltinFunction('reload', "O", "O", "PyImport_ReloadModule"), - BuiltinFunction('repr', "O", "O", "PyObject_Repr", builtin_return_type='str'), + BuiltinFunction('repr', "O", "O", "PyObject_Repr"), # , builtin_return_type='str'), # add in Cython 3.1 #('round', "", "", ""), BuiltinFunction('setattr', "OOO", "r", "PyObject_SetAttr"), #('sum', "", "", ""), #('sorted', "", "", ""), #('type', "O", "O", "PyObject_Type"), - #('unichr', "", "", ""), + BuiltinFunction('unichr', "l", "O", "PyUnicode_FromOrdinal", builtin_return_type='unicode'), #('unicode', "", "", ""), #('vars', "", "", ""), #('zip', "", "", ""), @@ -318,25 +331,30 @@ ("set", "PySet_Type", [BuiltinMethod("__contains__", "TO", "b", "PySequence_Contains"), BuiltinMethod("clear", "T", "r", "PySet_Clear"), # discard() and remove() have a special treatment for unhashable values -# BuiltinMethod("discard", "TO", "r", "PySet_Discard"), - BuiltinMethod("update", "TO", "r", "__Pyx_PySet_Update", - utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")), + BuiltinMethod("discard", "TO", "r", "__Pyx_PySet_Discard", + utility_code=UtilityCode.load("py_set_discard", "Optimize.c")), + BuiltinMethod("remove", "TO", "r", "__Pyx_PySet_Remove", + utility_code=UtilityCode.load("py_set_remove", "Optimize.c")), + # update is actually variadic (see Github issue #1645) +# BuiltinMethod("update", "TO", "r", "__Pyx_PySet_Update", +# utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")), BuiltinMethod("add", "TO", "r", "PySet_Add"), BuiltinMethod("pop", "T", "O", "PySet_Pop")]), ("frozenset", "PyFrozenSet_Type", []), ("Exception", "((PyTypeObject*)PyExc_Exception)[0]", []), + ("StopAsyncIteration", "((PyTypeObject*)__Pyx_PyExc_StopAsyncIteration)[0]", []), ] -types_that_construct_their_instance = set([ +types_that_construct_their_instance = frozenset({ # some builtin types do not always return an instance of # themselves - these do: 'type', 'bool', 'long', 'float', 'complex', 'bytes', 'unicode', 'bytearray', - 'tuple', 'list', 'dict', 'set', 'frozenset' + 'tuple', 'list', 'dict', 'set', 'frozenset', # 'str', # only in Py3.x # 'file', # only in Py2.x -]) +}) builtin_structs_table = [ @@ -376,10 +394,14 @@ utility = builtin_utility_code.get(name) if name == 'frozenset': objstruct_cname = 'PySetObject' + elif name == 'bytearray': + objstruct_cname = 'PyByteArrayObject' elif name == 'bool': objstruct_cname = None elif name == 'Exception': objstruct_cname = "PyBaseExceptionObject" + elif name == 'StopAsyncIteration': + objstruct_cname = "PyBaseExceptionObject" else: objstruct_cname = 'Py%sObject' % name.capitalize() the_type = builtin_scope.declare_builtin_type(name, cname, utility, objstruct_cname) @@ -398,6 +420,7 @@ def init_builtins(): + #Errors.init_thread() # hopefully not needed - we should not emit warnings ourselves init_builtin_structs() init_builtin_types() init_builtin_funcs() @@ -406,14 +429,9 @@ '__debug__', PyrexTypes.c_const_type(PyrexTypes.c_bint_type), pos=None, cname='(!Py_OptimizeFlag)', is_cdef=True) - entry = builtin_scope.declare_var( - 'StopAsyncIteration', PyrexTypes.py_object_type, - pos=None, cname='__Pyx_PyExc_StopAsyncIteration') - entry.utility_code = UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c") - global list_type, tuple_type, dict_type, set_type, frozenset_type global bytes_type, str_type, unicode_type, basestring_type, slice_type - global float_type, bool_type, type_type, complex_type, bytearray_type + global float_type, long_type, bool_type, type_type, complex_type, bytearray_type type_type = builtin_scope.lookup('type').type list_type = builtin_scope.lookup('list').type tuple_type = builtin_scope.lookup('tuple').type @@ -427,8 +445,62 @@ basestring_type = builtin_scope.lookup('basestring').type bytearray_type = builtin_scope.lookup('bytearray').type float_type = builtin_scope.lookup('float').type + long_type = builtin_scope.lookup('long').type bool_type = builtin_scope.lookup('bool').type complex_type = builtin_scope.lookup('complex').type init_builtins() + +############################## +# Support for a few standard library modules that Cython understands (currently typing and dataclasses) +############################## +_known_module_scopes = {} + +def get_known_standard_library_module_scope(module_name): + mod = _known_module_scopes.get(module_name) + if mod: + return mod + + if module_name == "typing": + mod = ModuleScope(module_name, None, None) + for name, tp in [ + ('Dict', dict_type), + ('List', list_type), + ('Tuple', tuple_type), + ('Set', set_type), + ('FrozenSet', frozenset_type), + ]: + name = EncodedString(name) + if name == "Tuple": + indexed_type = PyrexTypes.PythonTupleTypeConstructor(EncodedString("typing."+name), tp) + else: + indexed_type = PyrexTypes.PythonTypeConstructor(EncodedString("typing."+name), tp) + entry = mod.declare_type(name, indexed_type, pos = None) + + for name in ['ClassVar', 'Optional']: + indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("typing."+name)) + entry = mod.declare_type(name, indexed_type, pos = None) + _known_module_scopes[module_name] = mod + elif module_name == "dataclasses": + mod = ModuleScope(module_name, None, None) + indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("dataclasses.InitVar")) + entry = mod.declare_type(EncodedString("InitVar"), indexed_type, pos = None) + _known_module_scopes[module_name] = mod + return mod + + +def get_known_standard_library_entry(qualified_name): + name_parts = qualified_name.split(".") + module_name = EncodedString(name_parts[0]) + rest = name_parts[1:] + + if len(rest) > 1: # for now, we don't know how to deal with any nested modules + return None + + mod = get_known_standard_library_module_scope(module_name) + + # eventually handle more sophisticated multiple lookups if needed + if mod and rest: + return mod.lookup_here(rest[0]) + return None diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/CmdLine.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/CmdLine.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/CmdLine.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/CmdLine.py 2022-03-24 10:16:46.000000000 +0000 @@ -5,202 +5,222 @@ from __future__ import absolute_import import os -import sys +from argparse import ArgumentParser, Action, SUPPRESS from . import Options -usage = """\ -Cython (http://cython.org) is a compiler for code written in the -Cython language. Cython is based on Pyrex by Greg Ewing. - -Usage: cython [options] sourcefile.{pyx,py} ... - -Options: - -V, --version Display version number of cython compiler - -l, --create-listing Write error messages to a listing file - -I, --include-dir Search for include files in named directory - (multiple include directories are allowed). - -o, --output-file Specify name of generated C file - -t, --timestamps Only compile newer source files - -f, --force Compile all source files (overrides implied -t) - -v, --verbose Be verbose, print file names on multiple compilation - -p, --embed-positions If specified, the positions in Cython files of each - function definition is embedded in its docstring. - --cleanup Release interned objects on python exit, for memory debugging. - Level indicates aggressiveness, default 0 releases nothing. - -w, --working Sets the working directory for Cython (the directory modules - are searched from) - --gdb Output debug information for cygdb - --gdb-outdir Specify gdb debug information output directory. Implies --gdb. - - -D, --no-docstrings Strip docstrings from the compiled module. - -a, --annotate Produce a colorized HTML version of the source. - --annotate-coverage Annotate and include coverage information from cov.xml. - --line-directives Produce #line directives pointing to the .pyx source - --cplus Output a C++ rather than C file. - --embed[=] Generate a main() function that embeds the Python interpreter. - -2 Compile based on Python-2 syntax and code semantics. - -3 Compile based on Python-3 syntax and code semantics. - --lenient Change some compile time errors to runtime errors to - improve Python compatibility - --capi-reexport-cincludes Add cincluded headers to any auto-generated header files. - --fast-fail Abort the compilation on the first error - --warning-errors, -Werror Make all warnings into errors - --warning-extra, -Wextra Enable extra warnings - -X, --directive =[, for a name other than main().') + parser.add_argument('-2', dest='language_level', action='store_const', const=2, + help='Compile based on Python-2 syntax and code semantics.') + parser.add_argument('-3', dest='language_level', action='store_const', const=3, + help='Compile based on Python-3 syntax and code semantics.') + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', + help='Compile based on Python-3 syntax and code semantics without ' + 'assuming unicode by default for string literals under Python 2.') + parser.add_argument("--lenient", action=SetLenientAction, nargs=0, + help='Change some compile time errors to runtime errors to ' + 'improve Python compatibility') + parser.add_argument("--capi-reexport-cincludes", dest='capi_reexport_cincludes', action='store_true', + help='Add cincluded headers to any auto-generated header files.') + parser.add_argument("--fast-fail", dest='fast_fail', action='store_true', + help='Abort the compilation on the first error') + parser.add_argument("-Werror", "--warning-errors", dest='warning_errors', action='store_true', + help='Make all warnings into errors') + parser.add_argument("-Wextra", "--warning-extra", action=ActivateAllWarningsAction, nargs=0, + help='Enable extra warnings') + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='compiler_directives', type=str, + action=ParseDirectivesAction, + help='Overrides a compiler directive') + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', type=str, + action=ParseCompileTimeEnvAction, + help='Provides compile time env like DEF would do.') + parser.add_argument('sources', nargs='*', default=[]) + + # TODO: add help + parser.add_argument("-z", "--pre-import", dest='pre_import', action='store', type=str, help=SUPPRESS) + parser.add_argument("--convert-range", dest='convert_range', action='store_true', help=SUPPRESS) + parser.add_argument("--no-c-in-traceback", dest='c_line_in_traceback', action='store_false', help=SUPPRESS) + parser.add_argument("--cimport-from-pyx", dest='cimport_from_pyx', action='store_true', help=SUPPRESS) + parser.add_argument("--old-style-globals", dest='old_style_globals', action='store_true', help=SUPPRESS) + + # debug stuff: + from . import DebugFlags + for name in vars(DebugFlags): + if name.startswith("debug"): + option_name = name.replace('_', '-') + parser.add_argument("--" + option_name, action='store_true', help=SUPPRESS) + + return parser + + +def parse_command_line_raw(parser, args): + # special handling for --embed and --embed=xxxx as they aren't correctly parsed + def filter_out_embed_options(args): + with_embed, without_embed = [], [] + for x in args: + if x == '--embed' or x.startswith('--embed='): + with_embed.append(x) + else: + without_embed.append(x) + return with_embed, without_embed -def parse_command_line(args): - from .Main import CompilationOptions, default_options + with_embed, args_without_embed = filter_out_embed_options(args) - pending_arg = [] + arguments, unknown = parser.parse_known_args(args_without_embed) - def pop_arg(): - if not args or pending_arg: - bad_usage() - if '=' in args[0] and args[0].startswith('--'): # allow "--long-option=xyz" - name, value = args.pop(0).split('=', 1) - pending_arg.append(value) - return name - return args.pop(0) - - def pop_value(default=None): - if pending_arg: - return pending_arg.pop() - elif default is not None: - return default - elif not args: - bad_usage() - return args.pop(0) - - def get_param(option): - tail = option[2:] - if tail: - return tail + sources = arguments.sources + del arguments.sources + + # unknown can be either debug, embed or input files or really unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option " + option) else: - return pop_arg() + sources.append(option) - options = CompilationOptions(default_options) - sources = [] - while args: - if args[0].startswith("-"): - option = pop_arg() - if option in ("-V", "--version"): - options.show_version = 1 - elif option in ("-l", "--create-listing"): - options.use_listing_file = 1 - elif option in ("-+", "--cplus"): - options.cplus = 1 - elif option == "--embed": - Options.embed = pop_value("main") - elif option.startswith("-I"): - options.include_path.append(get_param(option)) - elif option == "--include-dir": - options.include_path.append(pop_value()) - elif option in ("-w", "--working"): - options.working_path = pop_value() - elif option in ("-o", "--output-file"): - options.output_file = pop_value() - elif option in ("-t", "--timestamps"): - options.timestamps = 1 - elif option in ("-f", "--force"): - options.timestamps = 0 - elif option in ("-v", "--verbose"): - options.verbose += 1 - elif option in ("-p", "--embed-positions"): - Options.embed_pos_in_docstring = 1 - elif option in ("-z", "--pre-import"): - Options.pre_import = pop_value() - elif option == "--cleanup": - Options.generate_cleanup_code = int(pop_value()) - elif option in ("-D", "--no-docstrings"): - Options.docstrings = False - elif option in ("-a", "--annotate"): - Options.annotate = True - elif option == "--annotate-coverage": - Options.annotate = True - Options.annotate_coverage_xml = pop_value() - elif option == "--convert-range": - Options.convert_range = True - elif option == "--line-directives": - options.emit_linenums = True - elif option == "--no-c-in-traceback": - options.c_line_in_traceback = False - elif option == "--gdb": - options.gdb_debug = True - options.output_dir = os.curdir - elif option == "--gdb-outdir": - options.gdb_debug = True - options.output_dir = pop_value() - elif option == "--lenient": - Options.error_on_unknown_names = False - Options.error_on_uninitialized = False - elif option == '-2': - options.language_level = 2 - elif option == '-3': - options.language_level = 3 - elif option == "--capi-reexport-cincludes": - options.capi_reexport_cincludes = True - elif option == "--fast-fail": - Options.fast_fail = True - elif option in ('-Werror', '--warning-errors'): - Options.warning_errors = True - elif option in ('-Wextra', '--warning-extra'): - options.compiler_directives.update(Options.extra_warnings) - elif option == "--old-style-globals": - Options.old_style_globals = True - elif option == "--directive" or option.startswith('-X'): - if option.startswith('-X') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compiler_directives = Options.parse_directive_list( - x_args, relaxed_bool=True, - current_settings=options.compiler_directives) - except ValueError as e: - sys.stderr.write("Error in compiler directive: %s\n" % e.args[0]) - sys.exit(1) - elif option.startswith('--debug'): - option = option[2:].replace('-', '_') - from . import DebugFlags - if option in dir(DebugFlags): - setattr(DebugFlags, option, True) - else: - sys.stderr.write("Unknown debug flag: %s\n" % option) - bad_usage() - elif option in ('-h', '--help'): - sys.stdout.write(usage) - sys.exit(0) - else: - sys.stderr.write("Unknown compiler flag: %s\n" % option) - sys.exit(1) + # embed-stuff must be handled extra: + for x in with_embed: + if x == '--embed': + name = 'main' # default value else: - sources.append(pop_arg()) + name = x[len('--embed='):] + setattr(arguments, 'embed', name) + + return arguments, sources - if pending_arg: - bad_usage() + +def parse_command_line(args): + parser = create_cython_argparser() + arguments, sources = parse_command_line_raw(parser, args) + + options = Options.CompilationOptions(Options.default_options) + for name, value in vars(arguments).items(): + if name.startswith('debug'): + from . import DebugFlags + if name in dir(DebugFlags): + setattr(DebugFlags, name, value) + else: + parser.error("Unknown debug flag: %s\n" % name) + elif hasattr(Options, name): + setattr(Options, name, value) + else: + setattr(options, name, value) if options.use_listing_file and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -o\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -o\n") if len(sources) == 0 and not options.show_version: - bad_usage() + parser.error("cython: Need at least one source file\n") if Options.embed and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -embed\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -embed\n") return options, sources - diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/CodeGeneration.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/CodeGeneration.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/CodeGeneration.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/CodeGeneration.py 2022-03-24 10:16:46.000000000 +0000 @@ -12,7 +12,7 @@ The result is a tuple (StatListNode, ModuleScope), i.e. everything that is needed from the pxd after it is processed. - A purer approach would be to seperately compile the pxd code, + A purer approach would be to separately compile the pxd code, but the result would have to be slightly more sophisticated than pure strings (functions + wanted interned strings + wanted utility code + wanted cached objects) so for now this diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Code.pxd cython-0.20.1+1~202203241016-9537/Cython/Compiler/Code.pxd --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Code.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Code.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -1,21 +1,27 @@ - -from __future__ import absolute_import +# cython: language_level=3 cimport cython +from ..StringIOTree cimport StringIOTree + + +cdef class UtilityCodeBase(object): + cpdef format_code(self, code_string, replace_empty_lines=*) + + +cdef class UtilityCode(UtilityCodeBase): + cdef public object name + cdef public object proto + cdef public object impl + cdef public object init + cdef public object cleanup + cdef public object proto_block + cdef public object requires + cdef public dict _cache + cdef public list specialize_list + cdef public object file + + cpdef none_or_sub(self, s, context) -#cdef class UtilityCodeBase(object): -# cdef public object name -# cdef public object proto -# cdef public object impl -# cdef public object init -# cdef public object cleanup -# cdef public object requires -# cdef public dict _cache -# cdef public list specialize_list -# cdef public object proto_block -# cdef public object file -# -# cpdef format_code(self, code_string, replace_empty_lines=*) cdef class FunctionState: cdef public set names_taken @@ -33,6 +39,7 @@ cdef public object return_from_error_cleanup_label # not used in __init__ ? cdef public object exc_vars + cdef public object current_except cdef public bint in_try_finally cdef public bint can_trace cdef public bint gil_owned @@ -40,6 +47,7 @@ cdef public list temps_allocated cdef public dict temps_free cdef public dict temps_used_type + cdef public set zombie_temps cdef public size_t temp_counter cdef public list collect_temps_stack @@ -88,7 +96,27 @@ #def funccontext_property(name): -#class CCodeWriter(object): +cdef class CCodeWriter(object): + cdef readonly StringIOTree buffer + cdef readonly list pyclass_stack + cdef readonly object globalstate + cdef readonly object funcstate + cdef object code_config + cdef object last_pos + cdef object last_marked_pos + cdef Py_ssize_t level + cdef public Py_ssize_t call_level # debug-only, see Nodes.py + cdef bint bol + + cpdef write(self, s) + cpdef put(self, code) + cpdef put_safe(self, code) + cpdef putln(self, code=*, bint safe=*) + @cython.final + cdef increase_indent(self) + @cython.final + cdef decrease_indent(self) + cdef class PyrexCodeWriter: cdef public object f diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Code.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Code.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Code.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Code.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ -# cython: language_level = 2 +# cython: language_level=3str +# cython: auto_pickle=False # # Code output module # @@ -6,31 +7,27 @@ from __future__ import absolute_import import cython -cython.declare(os=object, re=object, operator=object, - Naming=object, Options=object, StringEncoding=object, +cython.declare(os=object, re=object, operator=object, textwrap=object, + Template=object, Naming=object, Options=object, StringEncoding=object, Utils=object, SourceDescriptor=object, StringIOTree=object, - DebugFlags=object, basestring=object) + DebugFlags=object, basestring=object, defaultdict=object, + closing=object, partial=object) +import hashlib +import operator import os import re -import sys -import operator +import shutil import textwrap from string import Template from functools import partial from contextlib import closing from collections import defaultdict -try: - import hashlib -except ImportError: - import md5 as hashlib - from . import Naming from . import Options from . import DebugFlags from . import StringEncoding -from . import Version from .. import Utils from .Scanning import SourceDescriptor from ..StringIOTree import StringIOTree @@ -40,8 +37,6 @@ except ImportError: from builtins import str as basestring -KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) - non_portable_builtins_map = { # builtins that have different names in different Python versions @@ -52,30 +47,144 @@ 'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'), } +ctypedef_builtins_map = { + # types of builtins in "ctypedef class" statements which we don't + # import either because the names conflict with C types or because + # the type simply is not exposed. + 'py_int' : '&PyInt_Type', + 'py_long' : '&PyLong_Type', + 'py_float' : '&PyFloat_Type', + 'wrapper_descriptor' : '&PyWrapperDescr_Type', +} + basicsize_builtins_map = { # builtins whose type has a different tp_basicsize than sizeof(...) 'PyTypeObject': 'PyHeapTypeObject', } uncachable_builtins = [ - # builtin names that cannot be cached because they may or may not - # be available at import time + # Global/builtin names that cannot be cached because they may or may not + # be available at import time, for various reasons: + ## - Py3.7+ + 'breakpoint', # might deserve an implementation in Cython + ## - Py3.4+ + '__loader__', + '__spec__', + ## - Py3+ + 'BlockingIOError', + 'BrokenPipeError', + 'ChildProcessError', + 'ConnectionAbortedError', + 'ConnectionError', + 'ConnectionRefusedError', + 'ConnectionResetError', + 'FileExistsError', + 'FileNotFoundError', + 'InterruptedError', + 'IsADirectoryError', + 'ModuleNotFoundError', + 'NotADirectoryError', + 'PermissionError', + 'ProcessLookupError', + 'RecursionError', + 'ResourceWarning', + #'StopAsyncIteration', # backported + 'TimeoutError', + '__build_class__', + 'ascii', # might deserve an implementation in Cython + #'exec', # implemented in Cython + ## - platform specific 'WindowsError', - '_', # e.g. gettext + ## - others + '_', # e.g. used by gettext ] -special_py_methods = set([ +special_py_methods = cython.declare(frozenset, frozenset(( '__cinit__', '__dealloc__', '__richcmp__', '__next__', '__await__', '__aiter__', '__anext__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', - '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' -]) + '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', +))) modifier_output_mapper = { 'inline': 'CYTHON_INLINE' }.get -is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match + +class IncludeCode(object): + """ + An include file and/or verbatim C code to be included in the + generated sources. + """ + # attributes: + # + # pieces {order: unicode}: pieces of C code to be generated. + # For the included file, the key "order" is zero. + # For verbatim include code, the "order" is the "order" + # attribute of the original IncludeCode where this piece + # of C code was first added. This is needed to prevent + # duplication if the same include code is found through + # multiple cimports. + # location int: where to put this include in the C sources, one + # of the constants INITIAL, EARLY, LATE + # order int: sorting order (automatically set by increasing counter) + + # Constants for location. If the same include occurs with different + # locations, the earliest one takes precedense. + INITIAL = 0 + EARLY = 1 + LATE = 2 + + counter = 1 # Counter for "order" + + def __init__(self, include=None, verbatim=None, late=True, initial=False): + self.order = self.counter + type(self).counter += 1 + self.pieces = {} + + if include: + if include[0] == '<' and include[-1] == '>': + self.pieces[0] = u'#include {0}'.format(include) + late = False # system include is never late + else: + self.pieces[0] = u'#include "{0}"'.format(include) + + if verbatim: + self.pieces[self.order] = verbatim + + if initial: + self.location = self.INITIAL + elif late: + self.location = self.LATE + else: + self.location = self.EARLY + + def dict_update(self, d, key): + """ + Insert `self` in dict `d` with key `key`. If that key already + exists, update the attributes of the existing value with `self`. + """ + if key in d: + other = d[key] + other.location = min(self.location, other.location) + other.pieces.update(self.pieces) + else: + d[key] = self + + def sortkey(self): + return self.order + + def mainpiece(self): + """ + Return the main piece of C code, corresponding to the include + file. If there was no include file, return None. + """ + return self.pieces.get(0) + + def write(self, code): + # Write values of self.pieces dict, sorted by the keys + for k in sorted(self.pieces): + code.putln(self.pieces[k]) def get_utility_dir(): @@ -84,6 +193,28 @@ Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(Cython_dir, "Utility") +read_utilities_hook = None +""" +Override the hook for reading a utilities file that contains code fragments used +by the codegen. + +The hook functions takes the path of the utilities file, and returns a list +of strings, one per line. + +The default behavior is to open a file relative to get_utility_dir(). +""" + +def read_utilities_from_utility_dir(path): + """ + Read all lines of the file at the provided path from a path relative + to get_utility_dir(). + """ + filename = os.path.join(get_utility_dir(), path) + with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: + return f.readlines() + +# by default, read utilities from the utility directory. +read_utilities_hook = read_utilities_from_utility_dir class UtilityCodeBase(object): """ @@ -105,6 +236,15 @@ [definitions] + ##### MyUtility ##### + #@subsitute: tempita + + [requires tempita substitution + - context can't be specified here though so only + tempita utility that requires no external context + will benefit from this tag + - only necessary when @required from non-tempita code] + for prototypes and implementation respectively. For non-python or -cython files backslashes should be used instead. 5 to 30 comment characters may be used on either side. @@ -115,7 +255,6 @@ """ is_cython_utility = False - requires = None _utility_cache = {} @classmethod @@ -124,8 +263,7 @@ return code = '\n'.join(lines) - if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): - del tags['substitute'] + if tags and 'substitute' in tags and 'naming' in tags['substitute']: try: code = Template(code).substitute(vars(Naming)) except (KeyError, ValueError) as e: @@ -137,22 +275,15 @@ if type == 'proto': utility[0] = code - elif type.startswith('proto.'): - utility[0] = code - utility[1] = type[6:] elif type == 'impl': - utility[2] = code + utility[1] = code else: - all_tags = utility[3] - if KEYWORDS_MUST_BE_BYTES: - type = type.encode('ASCII') + all_tags = utility[2] all_tags[type] = code if tags: - all_tags = utility[3] + all_tags = utility[2] for name, values in tags.items(): - if KEYWORDS_MUST_BE_BYTES: - name = name.encode('ASCII') all_tags.setdefault(name, set()).update(values) @classmethod @@ -161,11 +292,10 @@ if utilities: return utilities - filename = os.path.join(get_utility_dir(), path) _, ext = os.path.splitext(path) if ext in ('.pyx', '.py', '.pxd', '.pxi'): comment = '#' - strip_comments = partial(re.compile(r'^\s*#.*').sub, '') + strip_comments = partial(re.compile(r'^\s*#(?!\s*cython\s*:).*').sub, '') rstrip = StringEncoding._unicode.rstrip else: comment = '/' @@ -175,12 +305,11 @@ (r'^%(C)s{5,30}\s*(?P(?:\w|\.)+)\s*%(C)s{5,30}|' r'^%(C)s+@(?P\w+)\s*:\s*(?P(?:\w|[.:])+)') % {'C': comment}).match - match_type = re.compile('(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match + match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match - with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: - all_lines = f.readlines() + all_lines = read_utilities_hook(path) - utilities = defaultdict(lambda: [None, None, None, {}]) + utilities = defaultdict(lambda: [None, None, {}]) lines = [] tags = defaultdict(set) utility = type = None @@ -220,43 +349,22 @@ return utilities @classmethod - def load(cls, util_code_name, from_file=None, **kwargs): + def load(cls, util_code_name, from_file, **kwargs): """ Load utility code from a file specified by from_file (relative to - Cython/Utility) and name util_code_name. If from_file is not given, - load it from the file util_code_name.*. There should be only one - file matched by this pattern. + Cython/Utility) and name util_code_name. """ + if '::' in util_code_name: from_file, util_code_name = util_code_name.rsplit('::', 1) - if not from_file: - utility_dir = get_utility_dir() - prefix = util_code_name + '.' - try: - listing = os.listdir(utility_dir) - except OSError: - # XXX the code below assumes as 'zipimport.zipimporter' instance - # XXX should be easy to generalize, but too lazy right now to write it - import zipfile - global __loader__ - loader = __loader__ - archive = loader.archive - with closing(zipfile.ZipFile(archive)) as fileobj: - listing = [os.path.basename(name) - for name in fileobj.namelist() - if os.path.join(archive, name).startswith(utility_dir)] - files = [filename for filename in listing - if filename.startswith(prefix)] - if not files: - raise ValueError("No match found for utility code " + util_code_name) - if len(files) > 1: - raise ValueError("More than one filename match found for utility code " + util_code_name) - from_file = files[0] - + assert from_file utilities = cls.load_utilities_from_file(from_file) - proto, proto_block, impl, tags = utilities[util_code_name] + proto, impl, tags = utilities[util_code_name] if tags: + if "substitute" in tags and "tempita" in tags["substitute"]: + if not issubclass(cls, TempitaUtilityCode): + return TempitaUtilityCode.load(util_code_name, from_file, **kwargs) orig_kwargs = kwargs.copy() for name, values in tags.items(): if name in kwargs: @@ -270,16 +378,20 @@ # dependencies are rarely unique, so use load_cached() when we can values = [cls.load_cached(dep, from_file) for dep in sorted(values)] + elif name == 'substitute': + # don't want to pass "naming" or "tempita" to the constructor + # since these will have been handled + values = values - {'naming', 'tempita'} + if not values: + continue elif not values: values = None elif len(values) == 1: - values = values[0] + values = list(values)[0] kwargs[name] = values if proto is not None: kwargs['proto'] = proto - if proto_block is not None: - kwargs['proto_block'] = proto_block if impl is not None: kwargs['impl'] = impl @@ -291,11 +403,11 @@ return cls(**kwargs) @classmethod - def load_cached(cls, utility_code_name, from_file=None, __cache={}): + def load_cached(cls, utility_code_name, from_file, __cache={}): """ Calls .load(), but using a per-type cache based on utility name and file name. """ - key = (cls, from_file, utility_code_name) + key = (utility_code_name, from_file, cls) try: return __cache[key] except KeyError: @@ -304,7 +416,7 @@ return code @classmethod - def load_as_string(cls, util_code_name, from_file=None, **kwargs): + def load_as_string(cls, util_code_name, from_file, **kwargs): """ Load a utility code as a string. Returns (proto, implementation) """ @@ -324,7 +436,11 @@ return "<%s(%s)>" % (type(self).__name__, self.name) def get_tree(self, **kwargs): - pass + return None + + def __deepcopy__(self, memodict=None): + # No need to deep-copy utility code since it's essentially immutable. + return self class UtilityCode(UtilityCodeBase): @@ -336,7 +452,7 @@ hashes/equals by instance proto C prototypes - impl implemenation code + impl implementation code init code to call on module initialization requires utility code dependencies proto_block the place in the resulting file where the prototype should @@ -384,9 +500,11 @@ def specialize(self, pyrex_type=None, **data): # Dicts aren't hashable... + name = self.name if pyrex_type is not None: data['type'] = pyrex_type.empty_declaration_code() data['type_name'] = pyrex_type.specialization_name() + name = "%s[%s]" % (name, data['type_name']) key = tuple(sorted(data.items())) try: return self._cache[key] @@ -402,7 +520,9 @@ self.none_or_sub(self.init, data), self.none_or_sub(self.cleanup, data), requires, - self.proto_block) + self.proto_block, + name, + ) self.specialize_list.append(s) return s @@ -410,22 +530,23 @@ def inject_string_constants(self, impl, output): """Replace 'PYIDENT("xyz")' by a constant Python identifier cname. """ - if 'PYIDENT(' not in impl: + if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl: return False, impl replacements = {} def externalise(matchobj): - name = matchobj.group(1) + key = matchobj.groups() try: - cname = replacements[name] + cname = replacements[key] except KeyError: - cname = replacements[name] = output.get_interned_identifier( - StringEncoding.EncodedString(name)).cname + str_type, name = key + cname = replacements[key] = output.get_py_string_const( + StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname return cname - impl = re.sub(r'PYIDENT\("([^"]+)"\)', externalise, impl) - assert 'PYIDENT(' not in impl - return bool(replacements), impl + impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl) + assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl + return True, impl def inject_unbound_methods(self, impl, output): """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname. @@ -433,28 +554,22 @@ if 'CALL_UNBOUND_METHOD(' not in impl: return False, impl - utility_code = set() def externalise(matchobj): - type_cname, method_name, args = matchobj.groups() - args = [arg.strip() for arg in args[1:].split(',')] - if len(args) == 1: - call = '__Pyx_CallUnboundCMethod0' - utility_code.add("CallUnboundCMethod0") - elif len(args) == 2: - call = '__Pyx_CallUnboundCMethod1' - utility_code.add("CallUnboundCMethod1") - else: - assert False, "CALL_UNBOUND_METHOD() requires 1 or 2 call arguments" - - cname = output.get_cached_unbound_method(type_cname, method_name, len(args)) - return '%s(&%s, %s)' % (call, cname, ', '.join(args)) - - impl = re.sub(r'CALL_UNBOUND_METHOD\(([a-zA-Z_]+),\s*"([^"]+)"((?:,\s*[^),]+)+)\)', externalise, impl) + type_cname, method_name, obj_cname, args = matchobj.groups() + args = [arg.strip() for arg in args[1:].split(',')] if args else [] + assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args) + return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args) + + impl = re.sub( + r'CALL_UNBOUND_METHOD\(' + r'([a-zA-Z_]+),' # type cname + r'\s*"([^"]+)",' # method name + r'\s*([^),]+)' # object cname + r'((?:,[^),]+)*)' # args* + r'\)', externalise, impl) assert 'CALL_UNBOUND_METHOD(' not in impl - for helper in sorted(utility_code): - output.use_utility_code(UtilityCode.load_cached(helper, "ObjectHandling.c")) - return bool(utility_code), impl + return True, impl def wrap_c_strings(self, impl): """Replace CSTRING('''xyz''') by a C compatible string @@ -563,6 +678,7 @@ available. Useful when you only have 'env' but not 'code'. """ __name__ = '' + requires = None def __init__(self, callback): self.callback = callback @@ -601,12 +717,14 @@ self.in_try_finally = 0 self.exc_vars = None + self.current_except = None self.can_trace = False self.gil_owned = True - self.temps_allocated = [] # of (name, type, manage_ref, static) - self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status - self.temps_used_type = {} # name -> (type, manage_ref) + self.temps_allocated = [] # of (name, type, manage_ref, static) + self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status + self.temps_used_type = {} # name -> (type, manage_ref) + self.zombie_temps = set() # temps that must not be reused after release self.temp_counter = 0 self.closure_temps = None @@ -621,6 +739,20 @@ self.should_declare_error_indicator = False self.uses_error_indicator = False + # safety checks + + def validate_exit(self): + # validate that all allocated temps have been freed + if self.temps_allocated: + leftovers = self.temps_in_use() + if leftovers: + msg = "TEMPGUARD: Temps left over at end of '%s': %s" % (self.scope.name, ', '.join([ + '%s [%s]' % (name, ctype) + for name, ctype, is_pytemp in sorted(leftovers)]), + ) + #print(msg) + raise RuntimeError(msg) + # labels def new_label(self, name=None): @@ -631,8 +763,8 @@ label += '_' + name return label - def new_yield_label(self): - label = self.new_label('resume_from_yield') + def new_yield_label(self, expr_type='yield'): + label = self.new_label('resume_from_%s' % expr_type) num_and_label = (len(self.yield_labels) + 1, label) self.yield_labels.append(num_and_label) return num_and_label @@ -690,7 +822,7 @@ # temp handling - def allocate_temp(self, type, manage_ref, static=False): + def allocate_temp(self, type, manage_ref, static=False, reusable=True): """ Allocates a temporary (which may create a new one or get a previously allocated and released one of the same type). Type is simply registered @@ -709,19 +841,26 @@ This is only used when allocating backing store for a module-level C array literals. + if reusable=False, the temp will not be reused after release. + A C string referring to the variable is returned. """ - if type.is_const and not type.is_reference: - type = type.const_base_type + if type.is_cv_qualified and not type.is_reference: + type = type.cv_base_type elif type.is_reference and not type.is_fake_reference: type = type.ref_base_type + elif type.is_cfunction: + from . import PyrexTypes + type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value + elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']: + self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp")) if not type.is_pyobject and not type.is_memoryviewslice: # Make manage_ref canonical, so that manage_ref will always mean # a decref is needed. manage_ref = False freelist = self.temps_free.get((type, manage_ref)) - if freelist is not None and freelist[0]: + if reusable and freelist is not None and freelist[0]: result = freelist[0].pop() freelist[1].remove(result) else: @@ -730,9 +869,11 @@ result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter) if result not in self.names_taken: break self.temps_allocated.append((result, type, manage_ref, static)) + if not reusable: + self.zombie_temps.add(result) self.temps_used_type[result] = (type, manage_ref) if DebugFlags.debug_temp_code_comments: - self.owner.putln("/* %s allocated (%s) */" % (result, type)) + self.owner.putln("/* %s allocated (%s)%s */" % (result, type, "" if reusable else " - zombie")) if self.collect_temps_stack: self.collect_temps_stack[-1].add((result, type)) @@ -751,10 +892,12 @@ self.temps_free[(type, manage_ref)] = freelist if name in freelist[1]: raise RuntimeError("Temp %s freed twice!" % name) - freelist[0].append(name) + if name not in self.zombie_temps: + freelist[0].append(name) freelist[1].add(name) if DebugFlags.debug_temp_code_comments: - self.owner.putln("/* %s released */" % name) + self.owner.putln("/* %s released %s*/" % ( + name, " - zombie" if name in self.zombie_temps else "")) def temps_in_use(self): """Return a list of (cname,type,manage_ref) tuples of temp names and their type @@ -774,7 +917,7 @@ """ return [(name, type) for name, type, manage_ref in self.temps_in_use() - if manage_ref and type.is_pyobject] + if manage_ref and type.is_pyobject] def all_managed_temps(self): """Return a list of (cname, type) tuples of refcount-managed Python objects. @@ -789,9 +932,11 @@ try-except and try-finally blocks to clean up temps in the error case. """ - return [(cname, type) - for (type, manage_ref), freelist in self.temps_free.items() if manage_ref - for cname in freelist[0]] + return sorted([ # Enforce deterministic order. + (cname, type) + for (type, manage_ref), freelist in self.temps_free.items() if manage_ref + for cname in freelist[0] + ]) def start_collecting_temps(self): """ @@ -977,29 +1122,45 @@ 'h_code', 'filename_table', 'utility_code_proto_before_types', - 'numeric_typedefs', # Let these detailed individual parts stay!, - 'complex_type_declarations', # as the proper solution is to make a full DAG... - 'type_declarations', # More coarse-grained blocks would simply hide - 'utility_code_proto', # the ugliness, not fix it + 'numeric_typedefs', # Let these detailed individual parts stay!, + 'complex_type_declarations', # as the proper solution is to make a full DAG... + 'type_declarations', # More coarse-grained blocks would simply hide + 'utility_code_proto', # the ugliness, not fix it 'module_declarations', 'typeinfo', 'before_global_var', 'global_var', 'string_decls', 'decls', - 'all_the_rest', + 'late_includes', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', # redefines names used in module_state/_clear/_traverse + 'module_code', # user code goes here 'pystring_table', 'cached_builtins', 'cached_constants', - 'init_globals', + 'init_constants', + 'init_globals', # (utility code called at init-time) 'init_module', 'cleanup_globals', 'cleanup_module', 'main_method', + 'utility_code_pragmas', # silence some irrelevant warnings in utility code 'utility_code_def', + 'utility_code_pragmas_end', # clean-up the utility_code_pragmas 'end' ] + # h files can only have a much smaller list of sections + h_code_layout = [ + 'h_code', + 'utility_code_proto_before_types', + 'type_declarations', + 'utility_code_proto', + 'end' + ] def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): self.filename_table = {} @@ -1011,42 +1172,51 @@ self.code_config = code_config self.common_utility_include_dir = common_utility_include_dir self.parts = {} - self.module_node = module_node # because some utility code generation needs it - # (generating backwards-compatible Get/ReleaseBuffer + self.module_node = module_node # because some utility code generation needs it + # (generating backwards-compatible Get/ReleaseBuffer self.const_cnames_used = {} self.string_const_index = {} + self.dedup_const_index = {} self.pyunicode_ptr_const_index = {} self.num_const_index = {} self.py_constants = [] self.cached_cmethods = {} + self.initialised_constants = set() writer.set_global_state(self) self.rootwriter = writer def initialize_main_c_code(self): rootwriter = self.rootwriter - for part in self.code_layout: - self.parts[part] = rootwriter.insertion_point() + for i, part in enumerate(self.code_layout): + w = self.parts[part] = rootwriter.insertion_point() + if i > 0: + w.putln("/* #### Code section: %s ### */" % part) if not Options.cache_builtins: del self.parts['cached_builtins'] else: w = self.parts['cached_builtins'] w.enter_cfunc_scope() - w.putln("static int __Pyx_InitCachedBuiltins(void) {") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) {") w = self.parts['cached_constants'] w.enter_cfunc_scope() w.putln("") - w.putln("static int __Pyx_InitCachedConstants(void) {") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") w.put_declare_refcount_context() - w.put_setup_refcount_context("__Pyx_InitCachedConstants") + w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants")) w = self.parts['init_globals'] w.enter_cfunc_scope() w.putln("") - w.putln("static int __Pyx_InitGlobals(void) {") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {") + + w = self.parts['init_constants'] + w.enter_cfunc_scope() + w.putln("") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitConstants(void) {") if not Options.generate_cleanup_code: del self.parts['cleanup_globals'] @@ -1054,7 +1224,7 @@ w = self.parts['cleanup_globals'] w.enter_cfunc_scope() w.putln("") - w.putln("static void __Pyx_CleanupGlobals(void) {") + w.putln("static CYTHON_SMALL_CODE void __Pyx_CleanupGlobals(void) {") code = self.parts['utility_code_proto'] code.putln("") @@ -1066,6 +1236,11 @@ code.putln("") code.putln("/* --- Runtime support code --- */") + def initialize_main_h_code(self): + rootwriter = self.rootwriter + for part in self.h_code_layout: + self.parts[part] = rootwriter.insertion_point() + def finalize_main_c_code(self): self.close_global_decls() @@ -1077,6 +1252,18 @@ code.put(util.format_code(util.impl)) code.putln("") + # + # utility code pragmas + # + code = self.parts['utility_code_pragmas'] + util = UtilityCode.load_cached("UtilityCodePragmas", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + code = self.parts['utility_code_pragmas_end'] + util = UtilityCode.load_cached("UtilityCodePragmasEnd", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + def __getitem__(self, key): return self.parts[key] @@ -1106,13 +1293,14 @@ w.putln("}") w.exit_cfunc_scope() - w = self.parts['init_globals'] - w.putln("return 0;") - if w.label_used(w.error_label): - w.put_label(w.error_label) - w.putln("return -1;") - w.putln("}") - w.exit_cfunc_scope() + for part in ['init_globals', 'init_constants']: + w = self.parts[part] + w.putln("return 0;") + if w.label_used(w.error_label): + w.put_label(w.error_label) + w.putln("return -1;") + w.putln("}") + w.exit_cfunc_scope() if Options.generate_cleanup_code: w = self.parts['cleanup_globals'] @@ -1129,7 +1317,12 @@ # constant handling at code generation time - def get_cached_constants_writer(self): + def get_cached_constants_writer(self, target=None): + if target is not None: + if target in self.initialised_constants: + # Return None on second/later calls to prevent duplicate creation code. + return None + self.initialised_constants.add(target) return self.parts['cached_constants'] def get_int_const(self, str_value, longness=False): @@ -1147,13 +1340,19 @@ c = self.new_num_const(str_value, 'float', value_code) return c - def get_py_const(self, type, prefix='', cleanup_level=None): + def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None): + if dedup_key is not None: + const = self.dedup_const_index.get(dedup_key) + if const is not None: + return const # create a new Python object constant const = self.new_py_const(type, prefix) if cleanup_level is not None \ and cleanup_level <= Options.generate_cleanup_code: cleanup_writer = self.parts['cleanup_globals'] cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) + if dedup_key is not None: + self.dedup_const_index[dedup_key] = const return const def get_string_const(self, text, py_version=None): @@ -1241,8 +1440,8 @@ prefix = Naming.const_prefix return "%s%s" % (prefix, name_suffix) - def get_cached_unbound_method(self, type_cname, method_name, args_count): - key = (type_cname, method_name, args_count) + def get_cached_unbound_method(self, type_cname, method_name): + key = (type_cname, method_name) try: cname = self.cached_cmethods[key] except KeyError: @@ -1250,6 +1449,18 @@ 'umethod', '%s_%s' % (type_cname, method_name)) return cname + def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames): + # admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ... + utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames) + self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c")) + cache_cname = self.get_cached_unbound_method(type_cname, method_name) + args = [obj_cname] + arg_cnames + return "__Pyx_%s(&%s, %s)" % ( + utility_code_name, + cache_cname, + ', '.join(args), + ) + def add_cached_builtin_decl(self, entry): if entry.is_builtin and entry.is_const: if self.should_declare(entry.cname, entry): @@ -1291,25 +1502,37 @@ for c in self.py_constants] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") for _, cname, c in consts: + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln( "static %s;" % c.type.declaration_code(cname)) + decls_writer.putln("#endif") def generate_cached_methods_decls(self): if not self.cached_cmethods: return decl = self.parts['decls'] - init = self.parts['init_globals'] + init = self.parts['init_constants'] cnames = [] - for (type_cname, method_name, _), cname in sorted(self.cached_cmethods.items()): + for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname - decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( - cname, method_name_cname)) + decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % ( + cname)) # split type reference storage as it might not be static init.putln('%s.type = (PyObject*)&%s;' % ( cname, type_cname)) + # method name string isn't static in limited api + init.putln('%s.method_name = &%s;' % ( + cname, method_name_cname)) if Options.generate_cleanup_code: cleanup = self.parts['cleanup_globals'] @@ -1347,13 +1570,26 @@ decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_constants = self.parts['init_constants'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + w.putln("#if CYTHON_USE_MODULE_STATE") + w_in_module_state = w.insertion_point() + w.putln("#else") + w_not_in_module_state = w.insertion_point() + w.putln("#endif") + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") + not_limited_api_decls_writer = decls_writer.insertion_point() + decls_writer.putln("#endif") + init_constants.putln("#if CYTHON_USE_MODULE_STATE") + init_constants_in_module_state = init_constants.insertion_point() + init_constants.putln("#endif") + for idx, py_string_args in enumerate(py_strings): + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1361,19 +1597,28 @@ else: encoding = '"%s"' % py_string.encoding.lower() - decls_writer.putln( + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) + not_limited_api_decls_writer.putln( "static PyObject *%s;" % py_string.cname) if py_string.py3str_cstring: - w.putln("#if PY_MAJOR_VERSION >= 3") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_in_module_state.putln("#if PY_MAJOR_VERSION >= 3") + w_not_in_module_state.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, py_string.py3str_cstring.cname, py_string.py3str_cstring.cname, '0', 1, 0, py_string.intern )) - w.putln("#else") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_in_module_state.putln("#else") + w_not_in_module_state.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, c_cname, c_cname, @@ -1383,24 +1628,46 @@ py_string.intern )) if py_string.py3str_cstring: - w.putln("#endif") + w_not_in_module_state.putln("#endif") + w_in_module_state.putln("{0, %s, sizeof(%s), %s, %d, %d, %d}," % ( + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + encoding if not py_string.py3str_cstring else '0', + py_string.is_unicode, + py_string.is_str, + py_string.intern + )) + init_constants_in_module_state.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % ( + Naming.stringtab_cname, + idx, + py_string.cname, + init_constants.error_goto(self.module_pos))) w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") - init_globals = self.parts['init_globals'] - init_globals.putln( + init_constants.putln("#if !CYTHON_USE_MODULE_STATE") + init_constants.putln( "if (__Pyx_InitStrings(%s) < 0) %s;" % ( Naming.stringtab_cname, - init_globals.error_goto(self.module_pos))) + init_constants.error_goto(self.module_pos))) + init_constants.putln("#endif") def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() decls_writer = self.parts['decls'] - init_globals = self.parts['init_globals'] + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") + init_constants = self.parts['init_constants'] for py_type, _, _, value, value_code, c in consts: cname = c.cname + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln("static PyObject *%s;" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' @@ -1412,9 +1679,10 @@ function = "PyInt_FromLong(%sL)" else: function = "PyInt_FromLong(%s)" - init_globals.putln('%s = %s; %s' % ( + init_constants.putln('%s = %s; %s' % ( cname, function % value_code, - init_globals.error_goto_if_null(cname, self.module_pos))) + init_constants.error_goto_if_null(cname, self.module_pos))) + decls_writer.putln("#endif") # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. @@ -1436,12 +1704,13 @@ # def lookup_filename(self, source_desc): + entry = source_desc.get_filenametable_entry() try: - index = self.filename_table[source_desc.get_filenametable_entry()] + index = self.filename_table[entry] except KeyError: index = len(self.filename_list) self.filename_list.append(source_desc) - self.filename_table[source_desc.get_filenametable_entry()] = index + self.filename_table[entry] = index return index def commented_file_contents(self, source_desc): @@ -1478,7 +1747,7 @@ See UtilityCode. """ - if utility_code not in self.utility_codes: + if utility_code and utility_code not in self.utility_codes: self.utility_codes.add(utility_code) utility_code.put_code(self) @@ -1491,7 +1760,8 @@ self.use_utility_code(entry.utility_code_definition) -def funccontext_property(name): +def funccontext_property(func): + name = func.__name__ attribute_of = operator.attrgetter(name) def get(self): return attribute_of(self.funcstate) @@ -1521,7 +1791,7 @@ as well - labels, temps, exc_vars: One must construct a scope in which these can exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for - sanity checking and forward compatabilty). Created insertion points + sanity checking and forward compatibility). Created insertion points looses this scope and cannot access it. - marker: Not copied to insertion point - filename_table, filename_list, input_file_contents: All codewriters @@ -1542,8 +1812,7 @@ # about the current class one is in # code_config CCodeConfig configuration options for the C code writer - globalstate = code_config = None - + @cython.locals(create_from='CCodeWriter') def __init__(self, create_from=None, buffer=None, copy_formatting=False): if buffer is None: buffer = StringIOTree() self.buffer = buffer @@ -1552,6 +1821,8 @@ self.pyclass_stack = [] self.funcstate = None + self.globalstate = None + self.code_config = None self.level = 0 self.call_level = 0 self.bol = 1 @@ -1586,10 +1857,13 @@ return self.buffer.getvalue() def write(self, s): - # also put invalid markers (lineno 0), to indicate that those lines - # have no Cython source code correspondence - cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 - self.buffer.markers.extend([cython_lineno] * s.count('\n')) + # Cygdb needs to know which Cython source line corresponds to which C line. + # Therefore, we write this information into "self.buffer.markers" and then write it from there + # into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map). + + filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0) + self.buffer.markers.extend([filename_line] * s.count('\n')) + self.buffer.write(s) def insertion_point(self): @@ -1614,19 +1888,27 @@ self.buffer.insert(writer.buffer) # Properties delegated to function scope - label_counter = funccontext_property("label_counter") - return_label = funccontext_property("return_label") - error_label = funccontext_property("error_label") - labels_used = funccontext_property("labels_used") - continue_label = funccontext_property("continue_label") - break_label = funccontext_property("break_label") - return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label") - yield_labels = funccontext_property("yield_labels") + @funccontext_property + def label_counter(self): pass + @funccontext_property + def return_label(self): pass + @funccontext_property + def error_label(self): pass + @funccontext_property + def labels_used(self): pass + @funccontext_property + def continue_label(self): pass + @funccontext_property + def break_label(self): pass + @funccontext_property + def return_from_error_cleanup_label(self): pass + @funccontext_property + def yield_labels(self): pass # Functions delegated to function scope def new_label(self, name=None): return self.funcstate.new_label(name) def new_error_label(self): return self.funcstate.new_error_label() - def new_yield_label(self): return self.funcstate.new_yield_label() + def new_yield_label(self, *args): return self.funcstate.new_yield_label(*args) def get_loop_labels(self): return self.funcstate.get_loop_labels() def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels) def new_loop_labels(self): return self.funcstate.new_loop_labels() @@ -1641,6 +1923,7 @@ self.funcstate = FunctionState(self, scope=scope) def exit_cfunc_scope(self): + self.funcstate.validate_exit() self.funcstate = None # constant handling @@ -1651,8 +1934,8 @@ def get_py_float(self, str_value, value_code): return self.globalstate.get_float_const(str_value, value_code).cname - def get_py_const(self, type, prefix='', cleanup_level=None): - return self.globalstate.get_py_const(type, prefix, cleanup_level).cname + def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None): + return self.globalstate.get_py_const(type, prefix, cleanup_level, dedup_key).cname def get_string_const(self, text): return self.globalstate.get_string_const(text).cname @@ -1674,8 +1957,8 @@ def intern_identifier(self, text): return self.get_py_string_const(text, identifier=True) - def get_cached_constants_writer(self): - return self.globalstate.get_cached_constants_writer() + def get_cached_constants_writer(self, target=None): + return self.globalstate.get_cached_constants_writer(target) # code generation @@ -1731,19 +2014,17 @@ include_dir = self.globalstate.common_utility_include_dir if include_dir and len(code) > 1024: include_file = "%s_%s.h" % ( - name, hashlib.md5(code.encode('utf8')).hexdigest()) + name, hashlib.sha1(code.encode('utf8')).hexdigest()) path = os.path.join(include_dir, include_file) if not os.path.exists(path): tmp_path = '%s.tmp%s' % (path, os.getpid()) with closing(Utils.open_new_file(tmp_path)) as f: f.write(code) - os.rename(tmp_path, path) + shutil.move(tmp_path, path) code = '#include "%s"\n' % path self.put(code) def put(self, code): - if is_self_assignment(code): - return fix_indent = False if "{" in code: dl = code.count("{") @@ -1815,22 +2096,29 @@ self.put("%s " % storage_class) if not entry.cf_used: self.put('CYTHON_UNUSED ') - self.put(entry.type.declaration_code( - entry.cname, dll_linkage=dll_linkage)) + if entry.is_cpp_optional: + self.put(entry.type.cpp_optional_declaration_code( + entry.cname, dll_linkage=dll_linkage)) + else: + self.put(entry.type.declaration_code( + entry.cname, dll_linkage=dll_linkage)) if entry.init is not None: self.put_safe(" = %s" % entry.type.literal_code(entry.init)) elif entry.type.is_pyobject: self.put(" = NULL") self.putln(";") + self.funcstate.scope.use_entry_utility_code(entry) def put_temp_declarations(self, func_context): for name, type, manage_ref, static in func_context.temps_allocated: - decl = type.declaration_code(name) + if type.is_cpp_class and not type.is_fake_reference and func_context.scope.directives['cpp_locals']: + decl = type.cpp_optional_declaration_code(name) + else: + decl = type.declaration_code(name) if type.is_pyobject: self.putln("%s = NULL;" % decl) elif type.is_memoryviewslice: - from . import MemoryView - self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) + self.putln("%s = %s;" % (decl, type.literal_code(type.default_value))) else: self.putln("%s%s;" % (static and "static " or "", decl)) @@ -1845,7 +2133,7 @@ self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) def put_generated_by(self): - self.putln("/* Generated by Cython %s */" % Version.watermark) + self.putln(Utils.GENERATED_BY_MARKER) self.putln("") def put_h_guard(self, guard): @@ -1868,7 +2156,7 @@ def entry_as_pyobject(self, entry): type = entry.type if (not entry.is_self_arg and not entry.type.is_complete() - or entry.type.is_extension_type): + or entry.type.is_extension_type): return "(PyObject *)" + entry.cname else: return entry.cname @@ -1877,117 +2165,89 @@ from .PyrexTypes import py_object_type, typecast return typecast(py_object_type, type, cname) - def put_gotref(self, cname): - self.putln("__Pyx_GOTREF(%s);" % cname) + def put_gotref(self, cname, type): + type.generate_gotref(self, cname) - def put_giveref(self, cname): - self.putln("__Pyx_GIVEREF(%s);" % cname) + def put_giveref(self, cname, type): + type.generate_giveref(self, cname) - def put_xgiveref(self, cname): - self.putln("__Pyx_XGIVEREF(%s);" % cname) + def put_xgiveref(self, cname, type): + type.generate_xgiveref(self, cname) - def put_xgotref(self, cname): - self.putln("__Pyx_XGOTREF(%s);" % cname) + def put_xgotref(self, cname, type): + type.generate_xgotref(self, cname) def put_incref(self, cname, type, nanny=True): - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) - else: - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + # Note: original put_Memslice_Incref/Decref also added in some utility code + # this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used + # and so has been removed. However, it's potentially a feature that might be useful here + type.generate_incref(self, cname, nanny=nanny) + + def put_xincref(self, cname, type, nanny=True): + type.generate_xincref(self, cname, nanny=nanny) + + def put_decref(self, cname, type, nanny=True, have_gil=True): + type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil) + + def put_xdecref(self, cname, type, nanny=True, have_gil=True): + type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil) + + def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) + + def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) + + def put_decref_set(self, cname, type, rhs_cname): + type.generate_decref_set(self, cname, rhs_cname) + + def put_xdecref_set(self, cname, type, rhs_cname): + type.generate_xdecref_set(self, cname, rhs_cname) - def put_decref(self, cname, type, nanny=True): - self._put_decref(cname, type, nanny, null_check=False, clear=False) + def put_incref_memoryviewslice(self, slice_cname, type, have_gil): + # TODO ideally this would just be merged into "put_incref" + type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil) + + def put_var_incref_memoryviewslice(self, entry, have_gil): + self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil) def put_var_gotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) + self.put_gotref(entry.cname, entry.type) def put_var_giveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) + self.put_giveref(entry.cname, entry.type) def put_var_xgotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) + self.put_xgotref(entry.cname, entry.type) def put_var_xgiveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) + self.put_xgiveref(entry.cname, entry.type) - def put_var_incref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_xincref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) - - def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=False, - clear=True, clear_before_decref=clear_before_decref) + def put_var_incref(self, entry, **kwds): + self.put_incref(entry.cname, entry.type, **kwds) - def put_xdecref(self, cname, type, nanny=True, have_gil=True): - self._put_decref(cname, type, nanny, null_check=True, - have_gil=have_gil, clear=False) + def put_var_xincref(self, entry, **kwds): + self.put_xincref(entry.cname, entry.type, **kwds) - def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=True, - clear=True, clear_before_decref=clear_before_decref) - - def _put_decref(self, cname, type, nanny=True, null_check=False, - have_gil=True, clear=False, clear_before_decref=False): - if type.is_memoryviewslice: - self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) - return + def put_var_decref(self, entry, **kwds): + self.put_decref(entry.cname, entry.type, **kwds) - prefix = nanny and '__Pyx' or 'Py' - X = null_check and 'X' or '' + def put_var_xdecref(self, entry, **kwds): + self.put_xdecref(entry.cname, entry.type, **kwds) - if clear: - if clear_before_decref: - if not nanny: - X = '' # CPython doesn't have a Py_XCLEAR() - self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) - else: - self.putln("%s_%sDECREF(%s); %s = 0;" % ( - prefix, X, self.as_pyobject(cname, type), cname)) - else: - self.putln("%s_%sDECREF(%s);" % ( - prefix, X, self.as_pyobject(cname, type))) + def put_var_decref_clear(self, entry, **kwds): + self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) - def put_decref_set(self, cname, rhs_cname): - self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_decref_set(self, entry, rhs_cname, **kwds): + self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds) - def put_xdecref_set(self, cname, rhs_cname): - self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) - - def put_var_decref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_xdecref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_decref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=False) - - def put_var_xdecref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=True) - - def _put_var_decref_clear(self, entry, null_check): - if entry.type.is_pyobject: - if entry.in_closure: - # reset before DECREF to make sure closure state is - # consistent during call to DECREF() - self.putln("__Pyx_%sCLEAR(%s);" % ( - null_check and 'X' or '', - entry.cname)) - else: - self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( - null_check and 'X' or '', - self.entry_as_pyobject(entry), - entry.cname)) + def put_var_xdecref_set(self, entry, rhs_cname, **kwds): + self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_clear(self, entry, **kwds): + self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) def put_var_decrefs(self, entries, used_only = 0): for entry in entries: @@ -2005,19 +2265,6 @@ for entry in entries: self.put_var_xdecref_clear(entry) - def put_incref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xgiveref_memoryviewslice(self, slice_cname): - self.put_xgiveref("%s.memview" % slice_cname) - def put_init_to_py_none(self, cname, type, nanny=True): from .PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") @@ -2034,7 +2281,7 @@ if entry.in_closure: self.put_giveref('Py_None') - def put_pymethoddef(self, entry, term, allow_skip=True): + def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None): if entry.is_special or entry.name == '__getattribute__': if entry.name not in special_py_methods: if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']: @@ -2044,25 +2291,47 @@ # that's better than ours. elif allow_skip: return - from .TypeSlots import method_coexist - if entry.doc: - doc_code = entry.doc_cname - else: - doc_code = 0 + method_flags = entry.signature.method_flags() - if method_flags: - if entry.is_special: - method_flags += [method_coexist] - self.putln( - '{"%s", (PyCFunction)%s, %s, %s}%s' % ( - entry.name, - entry.func_cname, - "|".join(method_flags), - doc_code, - term)) + if not method_flags: + return + if entry.is_special: + from . import TypeSlots + method_flags += [TypeSlots.method_coexist] + func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname + # Add required casts, but try not to shadow real warnings. + cast = entry.signature.method_function_type() + if cast != 'PyCFunction': + func_ptr = '(void*)(%s)%s' % (cast, func_ptr) + entry_name = entry.name.as_c_string_literal() + self.putln( + '{%s, (PyCFunction)%s, %s, %s}%s' % ( + entry_name, + func_ptr, + "|".join(method_flags), + entry.doc_cname if entry.doc else '0', + term)) + + def put_pymethoddef_wrapper(self, entry): + func_cname = entry.func_cname + if entry.is_special: + method_flags = entry.signature.method_flags() or [] + from .TypeSlots import method_noargs + if method_noargs in method_flags: + # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. + func_cname = Naming.method_wrapper_prefix + func_cname + self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( + func_cname, entry.func_cname)) + return func_cname # GIL methods + def use_fast_gil_utility_code(self): + if self.globalstate.directives['fast_gil']: + self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) + else: + self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + def put_ensure_gil(self, declare_gilstate=True, variable=None): """ Acquire the GIL. The generated code is safe even when no PyThreadState @@ -2072,42 +2341,59 @@ """ self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") if not variable: variable = '__pyx_gilstate_save' if declare_gilstate: self.put("PyGILState_STATE ") - self.putln("%s = PyGILState_Ensure();" % variable) + self.putln("%s = __Pyx_PyGILState_Ensure();" % variable) self.putln("#endif") def put_release_ensured_gil(self, variable=None): """ Releases the GIL, corresponds to `put_ensure_gil`. """ + self.use_fast_gil_utility_code() if not variable: variable = '__pyx_gilstate_save' self.putln("#ifdef WITH_THREAD") - self.putln("PyGILState_Release(%s);" % variable) + self.putln("__Pyx_PyGILState_Release(%s);" % variable) self.putln("#endif") - def put_acquire_gil(self, variable=None): + def put_acquire_gil(self, variable=None, unknown_gil_state=True): """ Acquire the GIL. The thread's thread state must have been initialized by a previous `put_release_gil` """ + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") + self.putln("__Pyx_FastGIL_Forget();") if variable: self.putln('_save = %s;' % variable) + if unknown_gil_state: + self.putln("if (_save) {") self.putln("Py_BLOCK_THREADS") + if unknown_gil_state: + self.putln("}") self.putln("#endif") - def put_release_gil(self, variable=None): + def put_release_gil(self, variable=None, unknown_gil_state=True): "Release the GIL, corresponds to `put_acquire_gil`." + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("PyThreadState *_save;") + self.putln("_save = NULL;") + if unknown_gil_state: + # we don't *know* that we don't have the GIL (since we may be inside a nogil function, + # and Py_UNBLOCK_THREADS is unsafe without the GIL) + self.putln("if (PyGILState_Check()) {") self.putln("Py_UNBLOCK_THREADS") + if unknown_gil_state: + self.putln("}") if variable: self.putln('%s = _save;' % variable) + self.putln("__Pyx_FastGIL_Remember();") self.putln("#endif") def declare_gilstate(self): @@ -2118,26 +2404,38 @@ # error handling def put_error_if_neg(self, pos, value): -# return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) # TODO this path is almost _never_ taken, yet this macro makes is slower! + # TODO this path is almost _never_ taken, yet this macro makes is slower! + # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) - def put_error_if_unbound(self, pos, entry, in_nogil_context=False): - from . import ExprNodes + def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None): if entry.from_closure: func = '__Pyx_RaiseClosureNameError' self.globalstate.use_utility_code( - ExprNodes.raise_closure_name_error_utility_code) + UtilityCode.load_cached("RaiseClosureNameError", "ObjectHandling.c")) elif entry.type.is_memoryviewslice and in_nogil_context: func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_memoryview_utility_code_nogil) + UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_cglobal: + func = '__Pyx_RaiseCppGlobalNameError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope: + # there doesn't seem to be a good way to detecting an instance-attribute of a C class + # (is_member is only set for class attributes) + func = '__Pyx_RaiseCppAttributeError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c")) else: func = '__Pyx_RaiseUnboundLocalError' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_local_error_utility_code) + UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c")) + if not unbound_check_code: + unbound_check_code = entry.type.check_for_null_code(entry.cname) self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( - entry.type.check_for_null_code(entry.cname), + unbound_check_code, func, entry.name, self.error_goto(pos))) @@ -2146,22 +2444,18 @@ self.funcstate.should_declare_error_indicator = True if used: self.funcstate.uses_error_indicator = True - if self.code_config.c_line_in_traceback: - cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro) - else: - cinfo = "" - - return "%s = %s[%s]; %s = %s;%s" % ( - Naming.filename_cname, - Naming.filetable_cname, + return "__PYX_MARK_ERR_POS(%s, %s)" % ( self.lookup_filename(pos[0]), - Naming.lineno_cname, - pos[1], - cinfo) + pos[1]) - def error_goto(self, pos): + def error_goto(self, pos, used=True): lbl = self.funcstate.error_label self.funcstate.use_label(lbl) + if pos is None: + return 'goto %s;' % lbl + self.funcstate.should_declare_error_indicator = True + if used: + self.funcstate.uses_error_indicator = True return "__PYX_ERR(%s, %s, %s)" % ( self.lookup_filename(pos[0]), pos[1], @@ -2174,7 +2468,8 @@ return self.error_goto_if("!%s" % cname, pos) def error_goto_if_neg(self, cname, pos): - return self.error_goto_if("%s < 0" % cname, pos) + # Add extra parentheses to silence clang warnings about constant conditions. + return self.error_goto_if("(%s < 0)" % cname, pos) def error_goto_if_PyErr(self, pos): return self.error_goto_if("PyErr_Occurred()", pos) @@ -2186,28 +2481,31 @@ self.putln('__Pyx_RefNannyDeclarations') def put_setup_refcount_context(self, name, acquire_gil=False): + name = name.as_c_string_literal() # handle unicode names if acquire_gil: self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) + self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0)) - def put_finish_refcount_context(self): - self.putln("__Pyx_RefNannyFinishContext();") + def put_finish_refcount_context(self, nogil=False): + self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();") - def put_add_traceback(self, qualified_name): + def put_add_traceback(self, qualified_name, include_cline=True): """ Build a Python traceback for propagating exceptions. qualified_name should be the qualified name of the function. """ + qualified_name = qualified_name.as_c_string_literal() # handle unicode names format_tuple = ( qualified_name, - Naming.clineno_cname, + Naming.clineno_cname if include_cline else 0, Naming.lineno_cname, Naming.filename_cname, ) + self.funcstate.uses_error_indicator = True - self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) + self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple) def put_unraisable(self, qualified_name, nogil=False): """ @@ -2270,6 +2568,7 @@ self.putln(" #define unlikely(x) __builtin_expect(!!(x), 0)") self.putln("#endif") + class PyrexCodeWriter(object): # f file output file # level int indentation level diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/CythonScope.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/CythonScope.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/CythonScope.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/CythonScope.py 2022-03-24 10:16:46.000000000 +0000 @@ -6,6 +6,7 @@ from .Errors import error from .Scanning import StringSourceDescriptor from . import MemoryView +from .StringEncoding import EncodedString class CythonScope(ModuleScope): @@ -26,6 +27,10 @@ cname='') entry.in_cinclude = True + def is_cpp(self): + # Allow C++ utility code in C++ contexts. + return self.context.cpp + def lookup_type(self, name): # This function should go away when types are all first-level objects. type = parse_basic_type(name) @@ -121,10 +126,26 @@ view_utility_scope = MemoryView.view_utility_code.declare_in_scope( self.viewscope, cython_scope=self, - whitelist=MemoryView.view_utility_whitelist) + allowlist=MemoryView.view_utility_allowlist) + + # Marks the types as being cython_builtin_type so that they can be + # extended from without Cython attempting to import cython.view + ext_types = [ entry.type + for entry in view_utility_scope.entries.values() + if entry.type.is_extension_type ] + for ext_type in ext_types: + ext_type.is_cython_builtin_type = 1 # self.entries["array"] = view_utility_scope.entries.pop("array") + # dataclasses scope + dc_str = EncodedString(u'dataclasses') + dataclassesscope = ModuleScope(dc_str, self, context=None) + self.declare_module(dc_str, dataclassesscope, pos=None).as_module = dataclassesscope + dataclassesscope.is_cython_builtin = True + dataclassesscope.pxd_file_loaded = True + # doesn't actually have any contents + def create_cython_scope(context): # One could in fact probably make it a singleton, diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Dataclass.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Dataclass.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Dataclass.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Dataclass.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,740 @@ +# functions to transform a c class into a dataclass + +from collections import OrderedDict +from textwrap import dedent +import operator + +from . import ExprNodes +from . import Nodes +from . import PyrexTypes +from . import Builtin +from . import Naming +from .Errors import error, warning +from .Code import UtilityCode, TempitaUtilityCode +from .Visitor import VisitorTransform +from .StringEncoding import EncodedString +from .TreeFragment import TreeFragment +from .ParseTreeTransforms import NormalizeTree, SkipDeclarations +from .Options import copy_inherited_directives + +_dataclass_loader_utilitycode = None + +def make_dataclasses_module_callnode(pos): + global _dataclass_loader_utilitycode + if not _dataclass_loader_utilitycode: + python_utility_code = UtilityCode.load_cached("Dataclasses_fallback", "Dataclasses.py") + python_utility_code = EncodedString(python_utility_code.impl) + _dataclass_loader_utilitycode = TempitaUtilityCode.load( + "SpecificModuleLoader", "Dataclasses.c", + context={'cname': "dataclasses", 'py_code': python_utility_code.as_c_string_literal()}) + return ExprNodes.PythonCapiCallNode( + pos, "__Pyx_Load_dataclasses_Module", + PyrexTypes.CFuncType(PyrexTypes.py_object_type, []), + utility_code=_dataclass_loader_utilitycode, + args=[], + ) + +_INTERNAL_DEFAULTSHOLDER_NAME = EncodedString('__pyx_dataclass_defaults') + + +class RemoveAssignmentsToNames(VisitorTransform, SkipDeclarations): + """ + Cython (and Python) normally treats + + class A: + x = 1 + + as generating a class attribute. However for dataclasses the `= 1` should be interpreted as + a default value to initialize an instance attribute with. + This transform therefore removes the `x=1` assignment so that the class attribute isn't + generated, while recording what it has removed so that it can be used in the initialization. + """ + def __init__(self, names): + super(RemoveAssignmentsToNames, self).__init__() + self.names = names + self.removed_assignments = {} + + def visit_CClassNode(self, node): + self.visitchildren(node) + return node + + def visit_PyClassNode(self, node): + return node # go no further + + def visit_FuncDefNode(self, node): + return node # go no further + + def visit_SingleAssignmentNode(self, node): + if node.lhs.is_name and node.lhs.name in self.names: + if node.lhs.name in self.removed_assignments: + warning(node.pos, ("Multiple assignments for '%s' in dataclass; " + "using most recent") % node.lhs.name, 1) + self.removed_assignments[node.lhs.name] = node.rhs + return [] + return node + + # I believe cascaded assignment is always a syntax error with annotations + # so there's no need to define visit_CascadedAssignmentNode + + def visit_Node(self, node): + self.visitchildren(node) + return node + + +class _MISSING_TYPE(object): + pass +MISSING = _MISSING_TYPE() + + +class Field(object): + """ + Field is based on the dataclasses.field class from the standard library module. + It is used internally during the generation of Cython dataclasses to keep track + of the settings for individual attributes. + + Attributes of this class are stored as nodes so they can be used in code construction + more readily (i.e. we store BoolNode rather than bool) + """ + default = MISSING + default_factory = MISSING + private = False + + literal_keys = ("repr", "hash", "init", "compare", "metadata") + + # default values are defined by the CPython dataclasses.field + def __init__(self, pos, default=MISSING, default_factory=MISSING, + repr=None, hash=None, init=None, + compare=None, metadata=None, + is_initvar=False, is_classvar=False, + **additional_kwds): + if default is not MISSING: + self.default = default + if default_factory is not MISSING: + self.default_factory = default_factory + self.repr = repr or ExprNodes.BoolNode(pos, value=True) + self.hash = hash or ExprNodes.NoneNode(pos) + self.init = init or ExprNodes.BoolNode(pos, value=True) + self.compare = compare or ExprNodes.BoolNode(pos, value=True) + self.metadata = metadata or ExprNodes.NoneNode(pos) + self.is_initvar = is_initvar + self.is_classvar = is_classvar + + for k, v in additional_kwds.items(): + # There should not be any additional keywords! + error(v.pos, "cython.dataclasses.field() got an unexpected keyword argument '%s'" % k) + + for field_name in self.literal_keys: + field_value = getattr(self, field_name) + if not field_value.is_literal: + error(field_value.pos, + "cython.dataclasses.field parameter '%s' must be a literal value" % field_name) + + def iterate_record_node_arguments(self): + for key in (self.literal_keys + ('default', 'default_factory')): + value = getattr(self, key) + if value is not MISSING: + yield key, value + + +def process_class_get_fields(node): + var_entries = node.scope.var_entries + # order of definition is used in the dataclass + var_entries = sorted(var_entries, key=operator.attrgetter('pos')) + var_names = [entry.name for entry in var_entries] + + # don't treat `x = 1` as an assignment of a class attribute within the dataclass + transform = RemoveAssignmentsToNames(var_names) + transform(node) + default_value_assignments = transform.removed_assignments + + if node.base_type and node.base_type.dataclass_fields: + fields = node.base_type.dataclass_fields.copy() + else: + fields = OrderedDict() + + for entry in var_entries: + name = entry.name + is_initvar = (entry.type.python_type_constructor_name == "dataclasses.InitVar") + # TODO - classvars aren't included in "var_entries" so are missed here + # and thus this code is never triggered + is_classvar = (entry.type.python_type_constructor_name == "typing.ClassVar") + if is_initvar or is_classvar: + entry.type = entry.type.resolve() # no longer need the special type + if name in default_value_assignments: + assignment = default_value_assignments[name] + if (isinstance(assignment, ExprNodes.CallNode) + and assignment.function.as_cython_attribute() == "dataclasses.field"): + # I believe most of this is well-enforced when it's treated as a directive + # but it doesn't hurt to make sure + if (not isinstance(assignment, ExprNodes.GeneralCallNode) + or not isinstance(assignment.positional_args, ExprNodes.TupleNode) + or assignment.positional_args.args + or not isinstance(assignment.keyword_args, ExprNodes.DictNode)): + error(assignment.pos, "Call to 'cython.dataclasses.field' must only consist " + "of compile-time keyword arguments") + continue + keyword_args = assignment.keyword_args.as_python_dict() + if 'default' in keyword_args and 'default_factory' in keyword_args: + error(assignment.pos, "cannot specify both default and default_factory") + continue + field = Field(node.pos, **keyword_args) + else: + if isinstance(assignment, ExprNodes.CallNode): + func = assignment.function + if ((func.is_name and func.name == "field") + or (func.is_attribute and func.attribute == "field")): + warning(assignment.pos, "Do you mean cython.dataclasses.field instead?", 1) + if assignment.type in [Builtin.list_type, Builtin.dict_type, Builtin.set_type]: + # The standard library module generates a TypeError at runtime + # in this situation. + # Error message is copied from CPython + error(assignment.pos, "mutable default for field {1} is not allowed: " + "use default_factory".format(assignment.type.name, name)) + + field = Field(node.pos, default=assignment) + else: + field = Field(node.pos) + field.is_initvar = is_initvar + field.is_classvar = is_classvar + if entry.visibility == "private": + field.private = True + fields[name] = field + node.entry.type.dataclass_fields = fields + return fields + + +def handle_cclass_dataclass(node, dataclass_args, analyse_decs_transform): + # default argument values from https://docs.python.org/3/library/dataclasses.html + kwargs = dict(init=True, repr=True, eq=True, + order=False, unsafe_hash=False, frozen=False) + if dataclass_args is not None: + if dataclass_args[0]: + error(node.pos, "cython.dataclasses.dataclass takes no positional arguments") + for k, v in dataclass_args[1].items(): + if k not in kwargs: + error(node.pos, + "cython.dataclasses.dataclass() got an unexpected keyword argument '%s'" % k) + if not isinstance(v, ExprNodes.BoolNode): + error(node.pos, + "Arguments passed to cython.dataclasses.dataclass must be True or False") + kwargs[k] = v + + fields = process_class_get_fields(node) + + dataclass_module = make_dataclasses_module_callnode(node.pos) + + # create __dataclass_params__ attribute. I try to use the exact + # `_DataclassParams` class defined in the standard library module if at all possible + # for maximum duck-typing compatibility. + dataclass_params_func = ExprNodes.AttributeNode(node.pos, obj=dataclass_module, + attribute=EncodedString("_DataclassParams")) + dataclass_params_keywords = ExprNodes.DictNode.from_pairs( + node.pos, + [ (ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)), + ExprNodes.BoolNode(node.pos, value=v)) + for k, v in kwargs.items() ]) + dataclass_params = ExprNodes.GeneralCallNode(node.pos, + function = dataclass_params_func, + positional_args = ExprNodes.TupleNode(node.pos, args=[]), + keyword_args = dataclass_params_keywords) + dataclass_params_assignment = Nodes.SingleAssignmentNode( + node.pos, + lhs = ExprNodes.NameNode(node.pos, name=EncodedString("__dataclass_params__")), + rhs = dataclass_params) + + dataclass_fields_stats = _set_up_dataclass_fields(node, fields, dataclass_module) + + stats = Nodes.StatListNode(node.pos, + stats=[dataclass_params_assignment] + dataclass_fields_stats) + + code_lines = [] + placeholders = {} + extra_stats = [] + for cl, ph, es in [ generate_init_code(kwargs['init'], node, fields), + generate_repr_code(kwargs['repr'], node, fields), + generate_eq_code(kwargs['eq'], node, fields), + generate_order_code(kwargs['order'], node, fields), + generate_hash_code(kwargs['unsafe_hash'], kwargs['eq'], kwargs['frozen'], node, fields) ]: + code_lines.append(cl) + placeholders.update(ph) + extra_stats.extend(extra_stats) + + code_lines = "\n".join(code_lines) + code_tree = TreeFragment(code_lines, level='c_class', pipeline=[NormalizeTree(node.scope)] + ).substitute(placeholders) + + stats.stats += (code_tree.stats + extra_stats) + + # turn off annotation typing, so all arguments to __init__ are accepted as + # generic objects and thus can accept _HAS_DEFAULT_FACTORY. + # Type conversion comes later + comp_directives = Nodes.CompilerDirectivesNode(node.pos, + directives=copy_inherited_directives(node.scope.directives, annotation_typing=False), + body=stats) + + comp_directives.analyse_declarations(node.scope) + # probably already in this scope, but it doesn't hurt to make sure + analyse_decs_transform.enter_scope(node, node.scope) + analyse_decs_transform.visit(comp_directives) + analyse_decs_transform.exit_scope() + + node.body.stats.append(comp_directives) + + +def generate_init_code(init, node, fields): + """ + All of these "generate_*_code" functions return a tuple of: + - code string + - placeholder dict (often empty) + - stat list (often empty) + which can then be combined later and processed once. + + Notes on CPython generated "__init__": + * Implemented in `_init_fn`. + * The use of the `dataclasses._HAS_DEFAULT_FACTORY` sentinel value as + the default argument for fields that need constructing with a factory + function is copied from the CPython implementation. (`None` isn't + suitable because it could also be a value for the user to pass.) + There's no real reason why it needs importing from the dataclasses module + though - it could equally be a value generated by Cython when the module loads. + * seen_default and the associated error message are copied directly from Python + * Call to user-defined __post_init__ function (if it exists) is copied from + CPython. + """ + if not init or node.scope.lookup_here("__init__"): + return "", {}, [] + # selfname behaviour copied from the cpython module + selfname = "__dataclass_self__" if "self" in fields else "self" + args = [selfname] + + placeholders = {} + placeholder_count = [0] + + # create a temp to get _HAS_DEFAULT_FACTORY + dataclass_module = make_dataclasses_module_callnode(node.pos) + has_default_factory = ExprNodes.AttributeNode( + node.pos, + obj=dataclass_module, + attribute=EncodedString("_HAS_DEFAULT_FACTORY") + ) + + def get_placeholder_name(): + while True: + name = "INIT_PLACEHOLDER_%d" % placeholder_count[0] + if (name not in placeholders + and name not in fields): + # make sure name isn't already used and doesn't + # conflict with a variable name (which is unlikely but possible) + break + placeholder_count[0] += 1 + return name + + default_factory_placeholder = get_placeholder_name() + placeholders[default_factory_placeholder] = has_default_factory + + function_body_code_lines = [] + + seen_default = False + for name, field in fields.items(): + if not field.init.value: + continue + entry = node.scope.lookup(name) + if entry.annotation: + annotation = u": %s" % entry.annotation.string.value + else: + annotation = u"" + assignment = u'' + if field.default is not MISSING or field.default_factory is not MISSING: + seen_default = True + if field.default_factory is not MISSING: + ph_name = default_factory_placeholder + else: + ph_name = get_placeholder_name() + placeholders[ph_name] = field.default # should be a node + assignment = u" = %s" % ph_name + elif seen_default: + error(entry.pos, ("non-default argument '%s' follows default argument " + "in dataclass __init__") % name) + return "", {}, [] + + args.append(u"%s%s%s" % (name, annotation, assignment)) + + if field.is_initvar: + continue + elif field.default_factory is MISSING: + if field.init.value: + function_body_code_lines.append(u" %s.%s = %s" % (selfname, name, name)) + else: + ph_name = get_placeholder_name() + placeholders[ph_name] = field.default_factory + if field.init.value: + # close to: + # def __init__(self, name=_PLACEHOLDER_VALUE): + # self.name = name_default_factory() if name is _PLACEHOLDER_VALUE else name + function_body_code_lines.append(u" %s.%s = %s() if %s is %s else %s" % ( + selfname, name, ph_name, name, default_factory_placeholder, name)) + else: + # still need to use the default factory to initialize + function_body_code_lines.append(u" %s.%s = %s()" + % (selfname, name, ph_name)) + + args = u", ".join(args) + func_def = u"def __init__(%s):" % args + + code_lines = [func_def] + (function_body_code_lines or ["pass"]) + + if node.scope.lookup("__post_init__"): + post_init_vars = ", ".join(name for name, field in fields.items() + if field.is_initvar) + code_lines.append(" %s.__post_init__(%s)" % (selfname, post_init_vars)) + return u"\n".join(code_lines), placeholders, [] + + +def generate_repr_code(repr, node, fields): + """ + The CPython implementation is just: + ['return self.__class__.__qualname__ + f"(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in fields]) + + ')"'], + + The only notable difference here is self.__class__.__qualname__ -> type(self).__name__ + which is because Cython currently supports Python 2. + """ + if not repr or node.scope.lookup("__repr__"): + return "", {}, [] + code_lines = ["def __repr__(self):"] + strs = [u"%s={self.%s!r}" % (name, name) + for name, field in fields.items() + if field.repr.value and not field.is_initvar] + format_string = u", ".join(strs) + code_lines.append(u' name = getattr(type(self), "__qualname__", type(self).__name__)') + code_lines.append(u" return f'{name}(%s)'" % format_string) + code_lines = u"\n".join(code_lines) + + return code_lines, {}, [] + + +def generate_cmp_code(op, funcname, node, fields): + if node.scope.lookup_here(funcname): + return "", {}, [] + + names = [name for name, field in fields.items() if (field.compare.value and not field.is_initvar)] + + if not names: + return "", {}, [] # no comparable types + + code_lines = [ + "def %s(self, other):" % funcname, + " cdef %s other_cast" % node.class_name, + " if isinstance(other, %s):" % node.class_name, + " other_cast = <%s>other" % node.class_name, + " else:", + " return NotImplemented" + ] + + # The Python implementation of dataclasses.py does a tuple comparison + # (roughly): + # return self._attributes_to_tuple() {op} other._attributes_to_tuple() + # + # For the Cython implementation a tuple comparison isn't an option because + # not all attributes can be converted to Python objects and stored in a tuple + # + # TODO - better diagnostics of whether the types support comparison before + # generating the code. Plus, do we want to convert C structs to dicts and + # compare them that way (I think not, but it might be in demand)? + checks = [] + for name in names: + checks.append("(self.%s %s other_cast.%s)" % ( + name, op, name)) + + if checks: + code_lines.append(" return " + " and ".join(checks)) + else: + if "=" in op: + code_lines.append(" return True") # "() == ()" is True + else: + code_lines.append(" return False") + + code_lines = u"\n".join(code_lines) + + return code_lines, {}, [] + + +def generate_eq_code(eq, node, fields): + if not eq: + return code_lines, {}, [] + return generate_cmp_code("==", "__eq__", node, fields) + + +def generate_order_code(order, node, fields): + if not order: + return "", {}, [] + code_lines = [] + placeholders = {} + stats = [] + for op, name in [("<", "__lt__"), + ("<=", "__le__"), + (">", "__gt__"), + (">=", "__ge__")]: + res = generate_cmp_code(op, name, node, fields) + code_lines.append(res[0]) + placeholders.update(res[1]) + stats.extend(res[2]) + return "\n".join(code_lines), placeholders, stats + + +def generate_hash_code(unsafe_hash, eq, frozen, node, fields): + """ + Copied from CPython implementation - the intention is to follow this as far as + is possible: + # +------------------- unsafe_hash= parameter + # | +----------- eq= parameter + # | | +--- frozen= parameter + # | | | + # v v v | | | + # | no | yes | <--- class has explicitly defined __hash__ + # +=======+=======+=======+========+========+ + # | False | False | False | | | No __eq__, use the base class __hash__ + # +-------+-------+-------+--------+--------+ + # | False | False | True | | | No __eq__, use the base class __hash__ + # +-------+-------+-------+--------+--------+ + # | False | True | False | None | | <-- the default, not hashable + # +-------+-------+-------+--------+--------+ + # | False | True | True | add | | Frozen, so hashable, allows override + # +-------+-------+-------+--------+--------+ + # | True | False | False | add | raise | Has no __eq__, but hashable + # +-------+-------+-------+--------+--------+ + # | True | False | True | add | raise | Has no __eq__, but hashable + # +-------+-------+-------+--------+--------+ + # | True | True | False | add | raise | Not frozen, but hashable + # +-------+-------+-------+--------+--------+ + # | True | True | True | add | raise | Frozen, so hashable + # +=======+=======+=======+========+========+ + # For boxes that are blank, __hash__ is untouched and therefore + # inherited from the base class. If the base is object, then + # id-based hashing is used. + + The Python implementation creates a tuple of all the fields, then hashes them. + This implementation creates a tuple of all the hashes of all the fields and hashes that. + The reason for this slight difference is to avoid to-Python conversions for anything + that Cython knows how to hash directly (It doesn't look like this currently applies to + anything though...). + """ + + hash_entry = node.scope.lookup_here("__hash__") + if hash_entry: + # TODO ideally assignment of __hash__ to None shouldn't trigger this + # but difficult to get the right information here + if unsafe_hash: + # error message taken from CPython dataclasses module + error(node.pos, "Cannot overwrite attribute __hash__ in class %s" % node.class_name) + return "", {}, [] + if not unsafe_hash: + if not eq: + return + if not frozen: + return "", {}, [Nodes.SingleAssignmentNode( + node.pos, + lhs=ExprNodes.NameNode(node.pos, name=EncodedString("__hash__")), + rhs=ExprNodes.NoneNode(node.pos), + )] + + names = [ + name for name, field in fields.items() + if (not field.is_initvar and + (field.compare.value if field.hash.value is None else field.hash.value)) + ] + if not names: + return "", {}, [] # nothing to hash + + # make a tuple of the hashes + tpl = u", ".join(u"hash(self.%s)" % name for name in names ) + + # if we're here we want to generate a hash + code_lines = dedent(u"""\ + def __hash__(self): + return hash((%s)) + """) % tpl + + return code_lines, {}, [] + + +def get_field_type(pos, entry): + """ + sets the .type attribute for a field + + Returns the annotation if possible (since this is what the dataclasses + module does). If not (for example, attributes defined with cdef) then + it creates a string fallback. + """ + if entry.annotation: + # Right now it doesn't look like cdef classes generate an + # __annotations__ dict, therefore it's safe to just return + # entry.annotation + # (TODO: remove .string if we ditch PEP563) + return entry.annotation.string + # If they do in future then we may need to look up into that + # to duplicating the node. The code below should do this: + #class_name_node = ExprNodes.NameNode(pos, name=entry.scope.name) + #annotations = ExprNodes.AttributeNode( + # pos, obj=class_name_node, + # attribute=EncodedString("__annotations__") + #) + #return ExprNodes.IndexNode( + # pos, base=annotations, + # index=ExprNodes.StringNode(pos, value=entry.name) + #) + else: + # it's slightly unclear what the best option is here - we could + # try to return PyType_Type. This case should only happen with + # attributes defined with cdef so Cython is free to make it's own + # decision + s = entry.type.declaration_code("", for_display=1) + return ExprNodes.StringNode(pos, value=s) + + +class FieldRecordNode(ExprNodes.ExprNode): + """ + __dataclass_fields__ contains a bunch of field objects recording how each field + of the dataclass was initialized (mainly corresponding to the arguments passed to + the "field" function). This node is used for the attributes of these field objects. + + If possible, coerces `arg` to a Python object. + Otherwise, generates a sensible backup string. + """ + subexprs = ['arg'] + + def __init__(self, pos, arg): + super(FieldRecordNode, self).__init__(pos, arg=arg) + + def analyse_types(self, env): + self.arg.analyse_types(env) + self.type = self.arg.type + return self + + def coerce_to_pyobject(self, env): + if self.arg.type.can_coerce_to_pyobject(env): + return self.arg.coerce_to_pyobject(env) + else: + # A string representation of the code that gave the field seems like a reasonable + # fallback. This'll mostly happen for "default" and "default_factory" where the + # type may be a C-type that can't be converted to Python. + return self._make_string() + + def _make_string(self): + from .AutoDocTransforms import AnnotationWriter + writer = AnnotationWriter(description="Dataclass field") + string = writer.write(self.arg) + return ExprNodes.StringNode(self.pos, value=EncodedString(string)) + + def generate_evaluation_code(self, code): + return self.arg.generate_evaluation_code(code) + + +def _set_up_dataclass_fields(node, fields, dataclass_module): + # For defaults and default_factories containing things like lambda, + # they're already declared in the class scope, and it creates a big + # problem if multiple copies are floating around in both the __init__ + # function, and in the __dataclass_fields__ structure. + # Therefore, create module-level constants holding these values and + # pass those around instead + # + # If possible we use the `Field` class defined in the standard library + # module so that the information stored here is as close to a regular + # dataclass as is possible. + variables_assignment_stats = [] + for name, field in fields.items(): + if field.private: + continue # doesn't appear in the public interface + for attrname in [ "default", "default_factory" ]: + field_default = getattr(field, attrname) + if field_default is MISSING or field_default.is_literal or field_default.is_name: + # some simple cases where we don't need to set up + # the variable as a module-level constant + continue + global_scope = node.scope.global_scope() + module_field_name = global_scope.mangle( + global_scope.mangle(Naming.dataclass_field_default_cname, node.class_name), + name) + # create an entry in the global scope for this variable to live + field_node = ExprNodes.NameNode(field_default.pos, name=EncodedString(module_field_name)) + field_node.entry = global_scope.declare_var(field_node.name, type=field_default.type or PyrexTypes.unspecified_type, + pos=field_default.pos, cname=field_node.name, is_cdef=1) + # replace the field so that future users just receive the namenode + setattr(field, attrname, field_node) + + variables_assignment_stats.append( + Nodes.SingleAssignmentNode(field_default.pos, lhs=field_node, rhs=field_default)) + + placeholders = {} + field_func = ExprNodes.AttributeNode(node.pos, obj=dataclass_module, + attribute=EncodedString("field")) + dc_fields = ExprNodes.DictNode(node.pos, key_value_pairs=[]) + dc_fields_namevalue_assignments = [] + + for name, field in fields.items(): + if field.private: + continue # doesn't appear in the public interface + type_placeholder_name = "PLACEHOLDER_%s" % name + placeholders[type_placeholder_name] = get_field_type( + node.pos, node.scope.entries[name] + ) + + # defining these make the fields introspect more like a Python dataclass + field_type_placeholder_name = "PLACEHOLDER_FIELD_TYPE_%s" % name + if field.is_initvar: + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD_INITVAR") + ) + elif field.is_classvar: + # TODO - currently this isn't triggered + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD_CLASSVAR") + ) + else: + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD") + ) + + dc_field_keywords = ExprNodes.DictNode.from_pairs( + node.pos, + [(ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)), + FieldRecordNode(node.pos, arg=v)) + for k, v in field.iterate_record_node_arguments()] + + ) + dc_field_call = ExprNodes.GeneralCallNode( + node.pos, function = field_func, + positional_args = ExprNodes.TupleNode(node.pos, args=[]), + keyword_args = dc_field_keywords) + dc_fields.key_value_pairs.append( + ExprNodes.DictItemNode( + node.pos, + key=ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(name)), + value=dc_field_call)) + dc_fields_namevalue_assignments.append( + dedent(u"""\ + __dataclass_fields__[{0!r}].name = {0!r} + __dataclass_fields__[{0!r}].type = {1} + __dataclass_fields__[{0!r}]._field_type = {2} + """).format(name, type_placeholder_name, field_type_placeholder_name)) + + dataclass_fields_assignment = \ + Nodes.SingleAssignmentNode(node.pos, + lhs = ExprNodes.NameNode(node.pos, + name=EncodedString("__dataclass_fields__")), + rhs = dc_fields) + + dc_fields_namevalue_assignments = u"\n".join(dc_fields_namevalue_assignments) + dc_fields_namevalue_assignments = TreeFragment(dc_fields_namevalue_assignments, + level="c_class", + pipeline=[NormalizeTree(None)]) + dc_fields_namevalue_assignments = dc_fields_namevalue_assignments.substitute(placeholders) + + return (variables_assignment_stats + + [dataclass_fields_assignment] + + dc_fields_namevalue_assignments.stats) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Errors.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Errors.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Errors.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Errors.py 2022-03-24 10:16:46.000000000 +0000 @@ -10,6 +10,14 @@ any_string_type = (bytes, str) import sys +from contextlib import contextmanager + +try: + from threading import local as _threadlocal +except ImportError: + class _threadlocal(object): pass + +threadlocal = _threadlocal() from ..Utils import open_new_file from . import DebugFlags @@ -23,6 +31,8 @@ class PyrexWarning(Exception): pass +class CannotSpecialize(PyrexError): + pass def context(position): source = position[0] @@ -59,11 +69,9 @@ self.message_only = message self.formatted_message = format_error(message, position) self.reported = False - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, self.formatted_message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (position, message) def __str__(self): @@ -73,8 +81,6 @@ def __init__(self, position = None, message = ""): self.position = position - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, format_position(position) + message) class InternalError(Exception): @@ -113,7 +119,7 @@ message += u'%s: %s' % (cause.__class__.__name__, cause) CompileError.__init__(self, pos, message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (pos, context, message, cause, stacktrace) class NoElementTreeInstalledException(PyrexError): @@ -121,35 +127,29 @@ implementation was found """ -listing_file = None -num_errors = 0 -echo_file = None - -def open_listing_file(path, echo_to_stderr = 1): +def open_listing_file(path, echo_to_stderr=True): # Begin a new error listing. If path is None, no file # is opened, the error counter is just reset. - global listing_file, num_errors, echo_file if path is not None: - listing_file = open_new_file(path) + threadlocal.cython_errors_listing_file = open_new_file(path) else: - listing_file = None + threadlocal.cython_errors_listing_file = None if echo_to_stderr: - echo_file = sys.stderr + threadlocal.cython_errors_echo_file = sys.stderr else: - echo_file = None - num_errors = 0 + threadlocal.cython_errors_echo_file = None + threadlocal.cython_errors_count = 0 def close_listing_file(): - global listing_file - if listing_file: - listing_file.close() - listing_file = None - -def report_error(err): - if error_stack: + if threadlocal.cython_errors_listing_file: + threadlocal.cython_errors_listing_file.close() + threadlocal.cython_errors_listing_file = None + +def report_error(err, use_stack=True): + error_stack = threadlocal.cython_errors_stack + if error_stack and use_stack: error_stack[-1].append(err) else: - global num_errors # See Main.py for why dual reporting occurs. Quick fix for now. if err.reported: return err.reported = True @@ -158,41 +158,50 @@ # Python <= 2.5 does this for non-ASCII Unicode exceptions line = format_error(getattr(err, 'message_only', "[unprintable exception message]"), getattr(err, 'position', None)) + u'\n' + listing_file = threadlocal.cython_errors_listing_file if listing_file: try: listing_file.write(line) except UnicodeEncodeError: listing_file.write(line.encode('ASCII', 'replace')) + echo_file = threadlocal.cython_errors_echo_file if echo_file: try: echo_file.write(line) except UnicodeEncodeError: echo_file.write(line.encode('ASCII', 'replace')) - num_errors += 1 + threadlocal.cython_errors_count += 1 if Options.fast_fail: raise AbortError("fatal errors") - def error(position, message): #print("Errors.error:", repr(position), repr(message)) ### if position is None: raise InternalError(message) err = CompileError(position, message) - if DebugFlags.debug_exception_on_error: raise Exception(err) # debug + if DebugFlags.debug_exception_on_error: raise Exception(err) # debug report_error(err) return err -LEVEL = 1 # warn about all errors level 1 or higher +LEVEL = 1 # warn about all errors level 1 or higher + +def _write_file_encode(file, line): + try: + file.write(line) + except UnicodeEncodeError: + file.write(line.encode('ascii', 'replace')) def message(position, message, level=1): if level < LEVEL: return warn = CompileWarning(position, message) - line = "note: %s\n" % warn + line = u"note: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -202,48 +211,76 @@ if Options.warning_errors and position: return error(position, message) warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn -_warn_once_seen = {} def warn_once(position, message, level=0): - if level < LEVEL or message in _warn_once_seen: + if level < LEVEL: + return + warn_once_seen = threadlocal.cython_errors_warn_once_seen + if message in warn_once_seen: return warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) - _warn_once_seen[message] = True + _write_file_encode(echo_file, line) + warn_once_seen[message] = True return warn # These functions can be used to momentarily suppress errors. -error_stack = [] - def hold_errors(): - error_stack.append([]) + errors = [] + threadlocal.cython_errors_stack.append(errors) + return errors + def release_errors(ignore=False): - held_errors = error_stack.pop() + held_errors = threadlocal.cython_errors_stack.pop() if not ignore: for err in held_errors: report_error(err) + def held_errors(): - return error_stack[-1] + return threadlocal.cython_errors_stack[-1] -# this module needs a redesign to support parallel cythonisation, but -# for now, the following works at least in sequential compiler runs +# same as context manager: + +@contextmanager +def local_errors(ignore=False): + errors = hold_errors() + try: + yield errors + finally: + release_errors(ignore=ignore) + + +# Keep all global state in thread local storage to support parallel cythonisation in distutils. + +def init_thread(): + threadlocal.cython_errors_count = 0 + threadlocal.cython_errors_listing_file = None + threadlocal.cython_errors_echo_file = None + threadlocal.cython_errors_warn_once_seen = set() + threadlocal.cython_errors_stack = [] def reset(): - _warn_once_seen.clear() - del error_stack[:] + threadlocal.cython_errors_warn_once_seen.clear() + del threadlocal.cython_errors_stack[:] + +def get_errors_count(): + return threadlocal.cython_errors_count diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/ExprNodes.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/ExprNodes.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/ExprNodes.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/ExprNodes.py 2022-03-24 10:16:46.000000000 +0000 @@ -7,7 +7,7 @@ import cython cython.declare(error=object, warning=object, warn_once=object, InternalError=object, CompileError=object, UtilityCode=object, TempitaUtilityCode=object, - StringEncoding=object, operator=object, + StringEncoding=object, operator=object, local_errors=object, report_error=object, Naming=object, Nodes=object, PyrexTypes=object, py_object_type=object, list_type=object, tuple_type=object, set_type=object, dict_type=object, unicode_type=object, str_type=object, bytes_type=object, type_type=object, @@ -16,32 +16,42 @@ bytearray_type=object, slice_type=object, _py_int_types=object, IS_PYTHON3=cython.bint) +import re import sys import copy import os.path import operator -from .Errors import error, warning, warn_once, InternalError, CompileError -from .Errors import hold_errors, release_errors, held_errors, report_error +from .Errors import ( + error, warning, InternalError, CompileError, report_error, local_errors, + CannotSpecialize) from .Code import UtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming from . import Nodes -from .Nodes import Node, utility_code_for_imports +from .Nodes import Node, utility_code_for_imports, SingleAssignmentNode from . import PyrexTypes -from .PyrexTypes import py_object_type, c_long_type, typecast, error_type, \ +from .PyrexTypes import py_object_type, typecast, error_type, \ unspecified_type from . import TypeSlots -from .Builtin import list_type, tuple_type, set_type, dict_type, type_type, \ - unicode_type, str_type, bytes_type, bytearray_type, basestring_type, slice_type +from .Builtin import ( + list_type, tuple_type, set_type, dict_type, type_type, + unicode_type, str_type, bytes_type, bytearray_type, basestring_type, + slice_type, long_type, +) from . import Builtin from . import Symtab from .. import Utils from .Annotate import AnnotationItem from . import Future from ..Debugging import print_call_chain -from .DebugFlags import debug_disposal_code, debug_temp_alloc, \ - debug_coercion +from .DebugFlags import debug_disposal_code, debug_coercion + +from .Pythran import (to_pythran, is_pythran_supported_type, is_pythran_supported_operation_type, + is_pythran_expr, pythran_func_type, pythran_binop_type, pythran_unaryop_type, has_np_pythran, + pythran_indexing_code, pythran_indexing_type, is_pythran_supported_node_or_none, pythran_type, + pythran_is_numpy_func_supported, pythran_get_func_include_file, pythran_functor) +from .PyrexTypes import PythranExpr try: from __builtin__ import basestring @@ -113,7 +123,6 @@ "Cannot convert 'char*' to unicode implicitly, decoding required"), } - def find_coercion_error(type_tuple, default, env): err = coercion_error_dict.get(type_tuple) if err is None: @@ -177,25 +186,84 @@ else: return item.infer_type(env) # if we're lucky, all items have the same type - item_types = set([item.infer_type(env) for item in seq_node.args]) + item_types = {item.infer_type(env) for item in seq_node.args} if len(item_types) == 1: return item_types.pop() return None + +def make_dedup_key(outer_type, item_nodes): + """ + Recursively generate a deduplication key from a sequence of values. + Includes Cython node types to work around the fact that (1, 2.0) == (1.0, 2), for example. + + @param outer_type: The type of the outer container. + @param item_nodes: A sequence of constant nodes that will be traversed recursively. + @return: A tuple that can be used as a dict key for deduplication. + """ + item_keys = [ + (py_object_type, None, type(None)) if node is None + # For sequences and their "mult_factor", see TupleNode. + else make_dedup_key(node.type, [node.mult_factor if node.is_literal else None] + node.args) if node.is_sequence_constructor + else make_dedup_key(node.type, (node.start, node.stop, node.step)) if node.is_slice + # For constants, look at the Python value type if we don't know the concrete Cython type. + else (node.type, node.constant_result, + type(node.constant_result) if node.type is py_object_type else None) if node.has_constant_result() + # IdentifierStringNode doesn't usually have a "constant_result" set because: + # 1. it doesn't usually have unicode_value + # 2. it's often created later in the compilation process after ConstantFolding + # but should be cacheable + else (node.type, node.value, node.unicode_value, "IdentifierStringNode") if isinstance(node, IdentifierStringNode) + else None # something we cannot handle => short-circuit below + for node in item_nodes + ] + if None in item_keys: + return None + return outer_type, tuple(item_keys) + + +# Returns a block of code to translate the exception, +# plus a boolean indicating whether to check for Python exceptions. def get_exception_handler(exception_value): if exception_value is None: - return "__Pyx_CppExn2PyErr();" + return "__Pyx_CppExn2PyErr();", False + elif (exception_value.type == PyrexTypes.c_char_type + and exception_value.value == '*'): + return "__Pyx_CppExn2PyErr();", True elif exception_value.type.is_pyobject: - return 'try { throw; } catch(const std::exception& exn) { PyErr_SetString(%s, exn.what()); } catch(...) { PyErr_SetNone(%s); }' % ( - exception_value.entry.cname, - exception_value.entry.cname) + return ( + 'try { throw; } catch(const std::exception& exn) {' + 'PyErr_SetString(%s, exn.what());' + '} catch(...) { PyErr_SetNone(%s); }' % ( + exception_value.entry.cname, + exception_value.entry.cname), + False) else: - return '%s(); if (!PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError , "Error converting c++ exception.");' % exception_value.entry.cname + return ( + '%s(); if (!PyErr_Occurred())' + 'PyErr_SetString(PyExc_RuntimeError, ' + '"Error converting c++ exception.");' % ( + exception_value.entry.cname), + False) + + +def maybe_check_py_error(code, check_py_exception, pos, nogil): + if check_py_exception: + if nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) + code.putln(code.error_goto_if("__Pyx_ErrOccurredWithGIL()", pos)) + else: + code.putln(code.error_goto_if("PyErr_Occurred()", pos)) -def translate_cpp_exception(code, pos, inside, exception_value, nogil): - raise_py_exception = get_exception_handler(exception_value) + +def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil): + raise_py_exception, check_py_exception = get_exception_handler(exception_value) code.putln("try {") code.putln("%s" % inside) + if py_result: + code.putln(code.error_goto_if_null(py_result, pos)) + maybe_check_py_error(code, check_py_exception, pos, nogil) code.putln("} catch(...) {") if nogil: code.put_ensure_gil(declare_gilstate=True) @@ -205,16 +273,32 @@ code.putln(code.error_goto(pos)) code.putln("}") +def needs_cpp_exception_conversion(node): + assert node.exception_check == "+" + if node.exception_value is None: + return True + # exception_value can be a NameNode + # (in which case it's used as a handler function and no conversion is needed) + if node.exception_value.is_name: + return False + # or a CharNode with a value of "*" + if isinstance(node.exception_value, CharNode) and node.exception_value.value == "*": + return True + # Most other const-nodes are disallowed after "+" by the parser + return False + + # Used to handle the case where an lvalue expression and an overloaded assignment # both have an exception declaration. -def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, - lhs_exc_val, assign_exc_val, nogil): - handle_lhs_exc = get_exception_handler(lhs_exc_val) - handle_assignment_exc = get_exception_handler(assign_exc_val) +def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, lhs_exc_val, assign_exc_val, nogil): + handle_lhs_exc, lhc_check_py_exc = get_exception_handler(lhs_exc_val) + handle_assignment_exc, assignment_check_py_exc = get_exception_handler(assign_exc_val) code.putln("try {") code.putln(lhs_type.declaration_code("__pyx_local_lvalue = %s;" % lhs_code)) + maybe_check_py_error(code, lhc_check_py_exc, pos, nogil) code.putln("try {") code.putln("__pyx_local_lvalue = %s;" % rhs_code) + maybe_check_py_error(code, assignment_check_py_exc, pos, nogil) # Catch any exception from the overloaded assignment. code.putln("} catch(...) {") if nogil: @@ -249,14 +333,18 @@ # Cached result of subexpr_nodes() # use_managed_ref boolean use ref-counted temps/assignments/etc. # result_is_used boolean indicates that the result will be dropped and the + # is_numpy_attribute boolean Is a Numpy module attribute # result_code/temp_result can safely be set to None + # annotation ExprNode or None PEP526 annotation for names or expressions result_ctype = None type = None + annotation = None temp_code = None - old_temp = None # error checker for multiple frees etc. - use_managed_ref = True # can be set by optimisation transforms + old_temp = None # error checker for multiple frees etc. + use_managed_ref = True # can be set by optimisation transforms result_is_used = True + is_numpy_attribute = False # The Analyse Expressions phase for expressions is split # into two sub-phases: @@ -387,6 +475,7 @@ saved_subexpr_nodes = None is_temp = False + has_temp_moved = False # if True then attempting to do anything but free the temp is invalid is_target = False is_starred = False @@ -394,11 +483,13 @@ child_attrs = property(fget=operator.attrgetter('subexprs')) + def analyse_annotations(self, env): + pass + def not_implemented(self, method_name): - print_call_chain(method_name, "not implemented") ### + print_call_chain(method_name, "not implemented") raise InternalError( - "%s.%s not implemented" % - (self.__class__.__name__, method_name)) + "%s.%s not implemented" % (self.__class__.__name__, method_name)) def is_lvalue(self): return 0 @@ -437,6 +528,29 @@ else: return self.calculate_result_code() + def _make_move_result_rhs(self, result, optional=False): + if optional and not (self.is_temp and self.type.is_cpp_class and not self.type.is_reference): + return result + self.has_temp_moved = True + return "{}({})".format("__PYX_STD_MOVE_IF_SUPPORTED" if optional else "std::move", result) + + def move_result_rhs(self): + return self._make_move_result_rhs(self.result(), optional=True) + + def move_result_rhs_as(self, type): + result = self.result_as(type) + if not (type.is_reference or type.needs_refcounting): + requires_move = type.is_rvalue_reference and self.is_temp + result = self._make_move_result_rhs(result, optional=not requires_move) + return result + + def pythran_result(self, type_=None): + if is_pythran_supported_node_or_none(self): + return to_pythran(self) + + assert(type_ is not None) + return to_pythran(self, type_) + def is_c_result_required(self): """ Subtypes may return False here if result temp allocation can be skipped. @@ -501,6 +615,9 @@ def analyse_target_declaration(self, env): error(self.pos, "Cannot assign to or delete this") + def analyse_assignment_expression_target_declaration(self, env): + error(self.pos, "Cannot use anything except a name in an assignment expression") + # ------------- Expression Analysis ---------------- def analyse_const_expression(self, env): @@ -546,7 +663,7 @@ def type_dependencies(self, env): # Returns the list of entries whose types must be determined # before the type of self can be inferred. - if hasattr(self, 'type') and self.type is not None: + if getattr(self, 'type', None) is not None: return () return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ()) @@ -555,12 +672,13 @@ # Differs from analyse_types as it avoids unnecessary # analysis of subexpressions, but can assume everything # in self.type_dependencies() has been resolved. - if hasattr(self, 'type') and self.type is not None: - return self.type - elif hasattr(self, 'entry') and self.entry is not None: - return self.entry.type - else: - self.not_implemented("infer_type") + type = getattr(self, 'type', None) + if type is not None: + return type + entry = getattr(self, 'entry', None) + if entry is not None: + return entry.type + self.not_implemented("infer_type") def nonlocally_immutable(self): # Returns whether this variable is a safe reference, i.e. @@ -587,6 +705,19 @@ # type, return that type, else None. return None + def analyse_as_specialized_type(self, env): + type = self.analyse_as_type(env) + if type and type.is_fused and env.fused_to_specific: + # while it would be nice to test "if entry.type in env.fused_to_specific" + # rather than try/catch this doesn't work reliably (mainly for nested fused types) + try: + return type.specialize(env.fused_to_specific) + except KeyError: + pass + if type and type.is_fused: + error(self.pos, "Type is not specific") + return type + def analyse_as_extension_type(self, env): # If this node can be interpreted as a reference to an # extension type or builtin type, return its type, else None. @@ -678,19 +809,20 @@ def make_owned_reference(self, code): """ - If result is a pyobject, make sure we own a reference to it. + Make sure we own a reference to result. If the result is in a temp, it is already a new reference. """ - if self.type.is_pyobject and not self.result_in_temp(): + if not self.result_in_temp(): code.put_incref(self.result(), self.ctype()) def make_owned_memoryviewslice(self, code): """ Make sure we own the reference to this memoryview slice. """ + # TODO ideally this would be shared with "make_owned_reference" if not self.result_in_temp(): - code.put_incref_memoryviewslice(self.result(), - have_gil=self.in_nogil_context) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) def generate_evaluation_code(self, code): # Generate code to evaluate this node and @@ -717,19 +849,17 @@ self.not_implemented("generate_result_code") def generate_disposal_code(self, code): + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) if self.is_temp: if self.type.is_string or self.type.is_pyunicode_ptr: # postponed from self.generate_evaluation_code() self.generate_subexpr_disposal_code(code) self.free_subexpr_temps(code) if self.result(): - if self.type.is_pyobject: - code.put_decref_clear(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=not self.in_nogil_context) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.in_nogil_context) else: # Already done if self.is_temp self.generate_subexpr_disposal_code(code) @@ -751,11 +881,15 @@ elif self.type.is_memoryviewslice: code.putln("%s.memview = NULL;" % self.result()) code.putln("%s.data = NULL;" % self.result()) + + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) else: self.generate_subexpr_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): # Stub method for nodes which are not legal as # the LHS of an assignment. An error will have # been reported earlier. @@ -781,6 +915,32 @@ def generate_function_definitions(self, env, code): pass + # ----Generation of small bits of reference counting -- + + def generate_decref_set(self, code, rhs): + code.put_decref_set(self.result(), self.ctype(), rhs) + + def generate_xdecref_set(self, code, rhs): + code.put_xdecref_set(self.result(), self.ctype(), rhs) + + def generate_gotref(self, code, handle_null=False, + maybe_null_extra_check=True): + if not (handle_null and self.cf_is_null): + if (handle_null and self.cf_maybe_null + and maybe_null_extra_check): + self.generate_xgotref(code) + else: + code.put_gotref(self.result(), self.ctype()) + + def generate_xgotref(self, code): + code.put_xgotref(self.result(), self.ctype()) + + def generate_giveref(self, code): + code.put_giveref(self.result(), self.ctype()) + + def generate_xgiveref(self, code): + code.put_xgiveref(self.result(), self.ctype()) + # ---------------- Annotation --------------------- def annotate(self, code): @@ -815,8 +975,8 @@ if used_as_reference and not src_type.is_reference: dst_type = dst_type.ref_base_type - if src_type.is_const: - src_type = src_type.const_base_type + if src_type.is_cv_qualified: + src_type = src_type.cv_base_type if src_type.is_fused or dst_type.is_fused: # See if we are coercing a fused function to a pointer to a @@ -835,6 +995,9 @@ if src_type.is_fused: error(self.pos, "Type is not specialized") + elif src_type.is_null_ptr and dst_type.is_ptr: + # NULL can be implicitly cast to any pointer type + return self else: error(self.pos, "Cannot coerce to a type that is not specialized") @@ -856,16 +1019,19 @@ elif not src_type.is_error: error(self.pos, "Cannot convert '%s' to memoryviewslice" % (src_type,)) - elif not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast, - copying=self.is_memview_copy_assignment): - if src.type.dtype.same_as(dst_type.dtype): - msg = "Memoryview '%s' not conformable to memoryview '%s'." - tup = src.type, dst_type - else: - msg = "Different base types for memoryviews (%s, %s)" - tup = src.type.dtype, dst_type.dtype + else: + if src.type.writable_needed: + dst_type.writable_needed = True + if not src.type.conforms_to(dst_type, broadcast=self.is_memview_broadcast, + copying=self.is_memview_copy_assignment): + if src.type.dtype.same_as(dst_type.dtype): + msg = "Memoryview '%s' not conformable to memoryview '%s'." + tup = src.type, dst_type + else: + msg = "Different base types for memoryviews (%s, %s)" + tup = src.type.dtype, dst_type.dtype - error(self.pos, msg % tup) + error(self.pos, msg % tup) elif dst_type.is_pyobject: if not src.type.is_pyobject: @@ -876,6 +1042,16 @@ if not src.type.subtype_of(dst_type): if src.constant_result is not None: src = PyTypeTestNode(src, dst_type, env) + elif is_pythran_expr(dst_type) and is_pythran_supported_type(src.type): + # We let the compiler decide whether this is valid + return src + elif is_pythran_expr(src.type): + if is_pythran_supported_type(dst_type): + # Match the case were a pythran expr is assigned to a value, or vice versa. + # We let the C++ compiler decide whether this is valid or not! + return src + # Else, we need to convert the Pythran expression to a Python object + src = CoerceToPyTypeNode(src, env, type=dst_type) elif src.type.is_pyobject: if used_as_reference and dst_type.is_cpp_class: warning( @@ -886,7 +1062,8 @@ and src_type != dst_type and dst_type.assignable_from(src_type)): src = CoerceToComplexNode(src, dst_type, env) - else: # neither src nor dst are py types + else: + # neither src nor dst are py types # Added the string comparison, since for c types that # is enough, but Cython gets confused when the types are # in different pxi files. @@ -928,11 +1105,11 @@ return self elif type.is_pyobject or type.is_int or type.is_ptr or type.is_float: return CoerceToBooleanNode(self, env) - elif type.is_cpp_class: + elif type.is_cpp_class and type.scope and type.scope.lookup("operator bool"): return SimpleCallNode( self.pos, function=AttributeNode( - self.pos, obj=self, attribute='operator bool'), + self.pos, obj=self, attribute=StringEncoding.EncodedString('operator bool')), args=[]).analyse_types(env) elif type.is_ctuple: bool_value = len(type.components) == 0 @@ -1006,6 +1183,15 @@ kwargs[attr_name] = value return cls(node.pos, **kwargs) + def get_known_standard_library_import(self): + """ + Gets the module.path that this node was imported from. + + Many nodes do not have one, or it is ambiguous, in which case + this function returns a false value. + """ + return None + class AtomicExprNode(ExprNode): # Abstract base class for expression nodes which have @@ -1024,6 +1210,7 @@ is_literal = 1 type = py_object_type + nogil_check = None def is_simple(self): return 1 @@ -1049,14 +1236,18 @@ constant_result = None - nogil_check = None - def compile_time_value(self, denv): return None def may_be_none(self): return True + def coerce_to(self, dst_type, env): + if not (dst_type.is_pyobject or dst_type.is_memoryviewslice or dst_type.is_error): + # Catch this error early and loudly. + error(self.pos, "Cannot assign None to %s" % dst_type) + return super(NoneNode, self).coerce_to(dst_type, env) + class EllipsisNode(PyConstNode): # '...' in a subscript list. @@ -1114,11 +1305,15 @@ def calculate_result_code(self): if self.type.is_pyobject: - return self.value and 'Py_True' or 'Py_False' + return 'Py_True' if self.value else 'Py_False' else: return str(int(self.value)) def coerce_to(self, dst_type, env): + if dst_type == self.type: + return self + if dst_type is py_object_type and self.type is Builtin.bool_type: + return self if dst_type.is_pyobject and self.type.is_int: return BoolNode( self.pos, value=self.value, @@ -1162,7 +1357,7 @@ unsigned = "" longness = "" - is_c_literal = None # unknown + is_c_literal = None # unknown def __init__(self, pos, **kwds): ExprNode.__init__(self, pos, **kwds) @@ -1288,7 +1483,6 @@ def compile_time_value(self, denv): return Utils.str_to_number(self.value) - class FloatNode(ConstNode): type = PyrexTypes.c_double_type @@ -1339,19 +1533,24 @@ type = PyrexTypes.parse_basic_type(name) if type is not None: return type - hold_errors() + + global_entry = env.global_scope().lookup(name) + if global_entry and global_entry.is_type and global_entry.type: + return global_entry.type + from .TreeFragment import TreeFragment - pos = (pos[0], pos[1], pos[2]-7) - try: - declaration = TreeFragment(u"sizeof(%s)" % name, name=pos[0].filename, initial_pos=pos) - except CompileError: - sizeof_node = None - else: - sizeof_node = declaration.root.stats[0].expr - sizeof_node = sizeof_node.analyse_types(env) - release_errors(ignore=True) - if isinstance(sizeof_node, SizeofTypeNode): - return sizeof_node.arg_type + with local_errors(ignore=True): + pos = (pos[0], pos[1], pos[2]-7) + try: + declaration = TreeFragment(u"sizeof(%s)" % name, name=pos[0].filename, initial_pos=pos) + except CompileError: + pass + else: + sizeof_node = declaration.root.stats[0].expr + if isinstance(sizeof_node, SizeofTypeNode): + sizeof_node = sizeof_node.analyse_types(env) + if isinstance(sizeof_node, SizeofTypeNode): + return sizeof_node.arg_type return None @@ -1405,7 +1604,7 @@ node.type = Builtin.bytes_type else: self.check_for_coercion_error(dst_type, env, fail=True) - return node + return node elif dst_type in (PyrexTypes.c_char_ptr_type, PyrexTypes.c_const_char_ptr_type): node.type = dst_type return node @@ -1414,8 +1613,10 @@ else PyrexTypes.c_char_ptr_type) return CastNode(node, dst_type) elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type): - node.type = dst_type - return node + # Exclude the case of passing a C string literal into a non-const C++ string. + if not dst_type.is_cpp_class or dst_type.is_const: + node.type = dst_type + return node # We still need to perform normal coerce_to processing on the # result, because we might be coercing to an extension type, @@ -1435,7 +1636,7 @@ self.result_code = result def get_constant_c_result_code(self): - return None # FIXME + return None # FIXME def calculate_result_code(self): return self.result_code @@ -1490,12 +1691,9 @@ if dst_type.is_string and self.bytes_value is not None: # special case: '-3' enforced unicode literal used in a # C char* context - return BytesNode(self.pos, value=self.bytes_value - ).coerce_to(dst_type, env) + return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env) if dst_type.is_pyunicode_ptr: - node = UnicodeNode(self.pos, value=self.value) - node.type = dst_type - return node + return UnicodeNode(self.pos, value=self.value, type=dst_type) error(self.pos, "Unicode literals do not support coercion to C types other " "than Py_UNICODE/Py_UCS4 (for characters) or Py_UNICODE* " @@ -1519,20 +1717,28 @@ def generate_evaluation_code(self, code): if self.type.is_pyobject: - if self.contains_surrogates(): - # surrogates are not really portable and cannot be + # FIXME: this should go away entirely! + # Since string_contains_lone_surrogates() returns False for surrogate pairs in Py2/UCS2, + # Py2 can generate different code from Py3 here. Let's hope we get away with claiming that + # the processing of surrogate pairs in code was always ambiguous and lead to different results + # on P16/32bit Unicode platforms. + if StringEncoding.string_contains_lone_surrogates(self.value): + # lone (unpaired) surrogates are not really portable and cannot be # decoded by the UTF-8 codec in Py3.3 self.result_code = code.get_py_const(py_object_type, 'ustring') - data_cname = code.get_pyunicode_ptr_const(self.value) - code = code.get_cached_constants_writer() - code.mark_pos(self.pos) - code.putln( - "%s = PyUnicode_FromUnicode(%s, (sizeof(%s) / sizeof(Py_UNICODE))-1); %s" % ( + data_cname = code.get_string_const( + StringEncoding.BytesLiteral(self.value.encode('unicode_escape'))) + const_code = code.get_cached_constants_writer(self.result_code) + if const_code is None: + return # already initialised + const_code.mark_pos(self.pos) + const_code.putln( + "%s = PyUnicode_DecodeUnicodeEscape(%s, sizeof(%s) - 1, NULL); %s" % ( self.result_code, data_cname, data_cname, - code.error_goto_if_null(self.result_code, self.pos))) - code.put_error_if_neg( + const_code.error_goto_if_null(self.result_code, self.pos))) + const_code.put_error_if_neg( self.pos, "__Pyx_PyUnicode_READY(%s)" % self.result_code) else: self.result_code = code.get_py_string_const(self.value) @@ -1628,15 +1834,15 @@ class ImagNode(AtomicExprNode): # Imaginary number literal # - # value float imaginary part + # value string imaginary part (float value) type = PyrexTypes.c_double_complex_type def calculate_constant_result(self): - self.constant_result = complex(0.0, self.value) + self.constant_result = complex(0.0, float(self.value)) def compile_time_value(self, denv): - return complex(0.0, self.value) + return complex(0.0, float(self.value)) def analyse_types(self, env): self.type.create_declaration_utility_code(env) @@ -1651,7 +1857,7 @@ node = ImagNode(self.pos, value=self.value) if dst_type.is_pyobject: node.is_temp = 1 - node.type = PyrexTypes.py_object_type + node.type = Builtin.complex_type # We still need to perform normal coerce_to processing on the # result, because we might be coercing to an extension type, # in which case a type test node will be needed. @@ -1672,7 +1878,7 @@ self.result(), float(self.value), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class NewExprNode(AtomicExprNode): @@ -1690,11 +1896,7 @@ self.type = error_type return self.cpp_check(env) - constructor = type.scope.lookup(u'') - if constructor is None: - func_type = PyrexTypes.CFuncType(type, [], exception_check='+') - type.scope.declare_cfunction(u'', func_type, self.pos) - constructor = type.scope.lookup(u'') + constructor = type.get_constructor(self.pos) self.class_type = type self.entry = constructor self.type = constructor.type @@ -1729,7 +1931,7 @@ is_name = True is_cython_module = False cython_attribute = None - lhs_of_first_assignment = False # TODO: remove me + lhs_of_first_assignment = False # TODO: remove me is_used_as_rvalue = 0 entry = None type_entry = None @@ -1808,6 +2010,66 @@ return super(NameNode, self).coerce_to(dst_type, env) + def declare_from_annotation(self, env, as_target=False): + """Implements PEP 526 annotation typing in a fairly relaxed way. + + Annotations are ignored for global variables. + All other annotations are stored on the entry in the symbol table. + String literals are allowed and not evaluated. + The ambiguous Python types 'int' and 'long' are not evaluated - the 'cython.int' form must be used instead. + """ + name = self.name + annotation = self.annotation + entry = self.entry or env.lookup_here(name) + if not entry: + # annotations never create global cdef names + if env.is_module_scope: + return + if ( + # name: "description" => not a type, but still a declared variable or attribute + annotation.expr.is_string_literal + # don't do type analysis from annotations if not asked to, but still collect the annotation + or not env.directives['annotation_typing'] + ): + atype = None + elif env.is_py_class_scope: + # For Python class scopes every attribute is a Python object + atype = py_object_type + else: + _, atype = annotation.analyse_type_annotation(env) + if atype is None: + atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type + if atype.is_fused and env.fused_to_specific: + try: + atype = atype.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) + atype = error_type + visibility = 'private' + if 'dataclasses.dataclass' in env.directives: + # handle "frozen" directive - full inspection of the dataclass directives happens + # in Dataclass.py + frozen_directive = None + dataclass_directive = env.directives['dataclasses.dataclass'] + if dataclass_directive: + dataclass_directive_kwds = dataclass_directive[1] + frozen_directive = dataclass_directive_kwds.get('frozen', None) + is_frozen = frozen_directive and frozen_directive.is_literal and frozen_directive.value + if atype.is_pyobject or atype.can_coerce_to_pyobject(env): + visibility = 'readonly' if is_frozen else 'public' + # If the object can't be coerced that's fine - we just don't create a property + if as_target and env.is_c_class_scope and not (atype.is_pyobject or atype.is_error): + # TODO: this will need revising slightly if annotated cdef attributes are implemented + atype = py_object_type + warning(annotation.pos, "Annotation ignored since class-level attributes must be Python objects. " + "Were you trying to set up an instance attribute?", 2) + entry = self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target, visibility=visibility) + # Even if the entry already exists, make sure we're supplying an annotation if we can. + if annotation and not entry.annotation: + entry.annotation = annotation + def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. # Returns the module scope, or None. @@ -1816,6 +2078,10 @@ entry = env.lookup(self.name) if entry and entry.as_module: return entry.as_module + if entry and entry.known_standard_library_import: + scope = Builtin.get_known_standard_library_module_scope(entry.known_standard_library_import) + if scope and scope.is_module_scope: + return scope return None def analyse_as_type(self, env): @@ -1830,6 +2096,10 @@ entry = env.lookup(self.name) if entry and entry.is_type: return entry.type + elif entry and entry.known_standard_library_import: + entry = Builtin.get_known_standard_library_entry(entry.known_standard_library_import) + if entry and entry.is_type: + return entry.type else: return None @@ -1845,8 +2115,26 @@ return None def analyse_target_declaration(self, env): + return self._analyse_target_declaration(env, is_assignment_expression=False) + + def analyse_assignment_expression_target_declaration(self, env): + return self._analyse_target_declaration(env, is_assignment_expression=True) + + def _analyse_target_declaration(self, env, is_assignment_expression): + self.is_target = True if not self.entry: - self.entry = env.lookup_here(self.name) + if is_assignment_expression: + self.entry = env.lookup_assignment_expression_target(self.name) + else: + self.entry = env.lookup_here(self.name) + if self.entry: + self.entry.known_standard_library_import = "" # already exists somewhere and so is now ambiguous + if not self.entry and self.annotation is not None: + # name : type = ... + is_dataclass = 'dataclasses.dataclass' in env.directives + # In a dataclass, an assignment should not prevent a name from becoming an instance attribute. + # Hence, "as_target = not is_dataclass". + self.declare_from_annotation(env, as_target=not is_dataclass) if not self.entry: if env.directives['warn.undeclared']: warning(self.pos, "implicit declaration of '%s'" % self.name, 1) @@ -1854,25 +2142,33 @@ type = unspecified_type else: type = py_object_type - self.entry = env.declare_var(self.name, type, self.pos) + if is_assignment_expression: + self.entry = env.declare_assignment_expression_target(self.name, type, self.pos) + else: + self.entry = env.declare_var(self.name, type, self.pos) if self.entry.is_declared_generic: self.result_ctype = py_object_type + if self.entry.as_module: + # cimported modules namespace can shadow actual variables + self.entry.is_variable = 1 def analyse_types(self, env): self.initialized_check = env.directives['initializedcheck'] - if self.entry is None: - self.entry = env.lookup(self.name) - if not self.entry: - self.entry = env.declare_builtin(self.name, self.pos) - if not self.entry: - self.type = PyrexTypes.error_type - return self entry = self.entry - if entry: - entry.used = 1 - if entry.type.is_buffer: - from . import Buffer - Buffer.used_buffer_aux_vars(entry) + if entry is None: + entry = env.lookup(self.name) + if not entry: + entry = env.declare_builtin(self.name, self.pos) + if entry and entry.is_builtin and entry.is_const: + self.is_literal = True + if not entry: + self.type = PyrexTypes.error_type + return self + self.entry = entry + entry.used = 1 + if entry.type.is_buffer: + from . import Buffer + Buffer.used_buffer_aux_vars(entry) self.analyse_rvalue_entry(env) return self @@ -1889,8 +2185,6 @@ if self.type.is_const: error(self.pos, "Assignment to const '%s'" % self.name) - if self.type.is_reference: - error(self.pos, "Assignment to reference '%s'" % self.name) if not self.is_lvalue(): error(self.pos, "Assignment to non-lvalue '%s'" % self.name) self.type = PyrexTypes.error_type @@ -1927,7 +2221,7 @@ if self.is_used_as_rvalue: entry = self.entry if entry.is_builtin: - if not entry.is_const: # cached builtins are ok + if not entry.is_const: # cached builtins are ok self.gil_error() elif entry.is_pyglobal: self.gil_error() @@ -1952,19 +2246,24 @@ entry = self.entry if entry.is_type and entry.type.is_extension_type: self.type_entry = entry - if entry.is_type and entry.type.is_enum: + if entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum): py_entry = Symtab.Entry(self.name, None, py_object_type) py_entry.is_pyglobal = True py_entry.scope = self.entry.scope self.entry = py_entry - elif not (entry.is_const or entry.is_variable - or entry.is_builtin or entry.is_cfunction - or entry.is_cpp_class): - if self.entry.as_variable: - self.entry = self.entry.as_variable - else: - error(self.pos, - "'%s' is not a constant, variable or function identifier" % self.name) + elif not (entry.is_const or entry.is_variable or + entry.is_builtin or entry.is_cfunction or + entry.is_cpp_class): + if self.entry.as_variable: + self.entry = self.entry.as_variable + elif not self.is_cython_module: + error(self.pos, "'%s' is not a constant, variable or function identifier" % self.name) + + def is_cimported_module_without_shadow(self, env): + if self.is_cython_module or self.cython_attribute: + return False + entry = self.entry or env.lookup(self.name) + return entry.as_module and not entry.is_variable def is_simple(self): # If it's not a C variable, it'll be in a temp. @@ -1973,7 +2272,7 @@ def may_be_none(self): if self.cf_state and self.type and (self.type.is_pyobject or self.type.is_memoryviewslice): - # gard against infinite recursion on self-dependencies + # guard against infinite recursion on self-dependencies if getattr(self, '_none_checking', False): # self-dependency - either this node receives a None # value from *another* node, or it can not reference @@ -2004,7 +2303,11 @@ def check_const(self): entry = self.entry - if entry is not None and not (entry.is_const or entry.is_cfunction or entry.is_builtin): + if entry is not None and not ( + entry.is_const or + entry.is_cfunction or + entry.is_builtin or + entry.type.is_const): self.not_const() return False return True @@ -2036,22 +2339,25 @@ def calculate_result_code(self): entry = self.entry if not entry: - return "" # There was an error earlier + return "" # There was an error earlier + if self.entry.is_cpp_optional and not self.is_target: + return "(*%s)" % entry.cname return entry.cname def generate_result_code(self, code): - assert hasattr(self, 'entry') entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier + if entry.utility_code: + code.globalstate.use_utility_code(entry.utility_code) if entry.is_builtin and entry.is_const: - return # Lookup already cached + return # Lookup already cached elif entry.is_pyclass_attr: assert entry.type.is_pyobject, "Python global or builtin not a Python object" interned_cname = code.intern_identifier(self.entry.name) if entry.is_builtin: namespace = Naming.builtins_cname - else: # entry.is_pyglobal + else: # entry.is_pyglobal namespace = entry.scope.namespace_cname if not self.cf_is_null: code.putln( @@ -2064,13 +2370,13 @@ code.globalstate.use_utility_code( UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c")) code.putln( - '%s = __Pyx_GetModuleGlobalName(%s);' % ( + '__Pyx_GetModuleGlobalName(%s, %s);' % ( self.result(), interned_cname)) if not self.cf_is_null: code.putln("}") code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_builtin and not entry.scope.is_module_scope: # known builtin @@ -2083,7 +2389,7 @@ self.result(), interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_pyglobal or (entry.is_builtin and entry.scope.is_module_scope): # name in class body, global name or unknown builtin @@ -2093,7 +2399,7 @@ code.globalstate.use_utility_code( UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c")) code.putln( - '%s = __Pyx_GetModuleGlobalName(%s); %s' % ( + '__Pyx_GetModuleGlobalName(%s, %s); %s' % ( self.result(), interned_cname, code.error_goto_if_null(self.result(), self.pos))) @@ -2102,30 +2408,39 @@ code.globalstate.use_utility_code( UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c")) code.putln( - '%s = __Pyx_GetNameInClass(%s, %s); %s' % ( + '__Pyx_GetNameInClass(%s, %s, %s); %s' % ( self.result(), entry.scope.namespace_cname, interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice: # Raise UnboundLocalError for objects and memoryviewslices raise_unbound = ( (self.cf_maybe_null or self.cf_is_null) and not self.allow_null) - null_code = entry.type.check_for_null_code(entry.cname) memslice_check = entry.type.is_memoryviewslice and self.initialized_check + optional_cpp_check = entry.is_cpp_optional and self.initialized_check + + if optional_cpp_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + else: + unbound_check_code = entry.type.check_for_null_code(entry.cname) - if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check): - code.put_error_if_unbound(self.pos, entry, self.in_nogil_context) + if unbound_check_code and raise_unbound and (entry.type.is_pyobject or memslice_check or optional_cpp_check): + code.put_error_if_unbound(self.pos, entry, self.in_nogil_context, unbound_check_code=unbound_check_code) + + elif entry.is_cglobal and entry.is_cpp_optional and self.initialized_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + code.put_error_if_unbound(self.pos, entry, unbound_check_code=unbound_check_code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): #print "NameNode.generate_assignment_code:", self.name ### entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if (self.entry.type.is_ptr and isinstance(rhs, ListNode) and not self.lhs_of_first_assignment and not rhs.in_module_scope): @@ -2146,7 +2461,10 @@ setter = 'PyDict_SetItem' namespace = Naming.moddict_cname elif entry.is_pyclass_attr: - setter = 'PyObject_SetItem' + # Special-case setting __new__ + n = "SetNewInClass" if self.name == "__new__" else "SetNameInClass" + code.globalstate.use_utility_code(UtilityCode.load_cached(n, "ObjectHandling.c")) + setter = '__Pyx_' + n else: assert False, repr(entry) code.put_error_if_neg( @@ -2188,38 +2506,39 @@ rhs.make_owned_reference(code) is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure if is_external_ref: - if not self.cf_is_null: - if self.cf_maybe_null: - code.put_xgotref(self.py_result()) - else: - code.put_gotref(self.py_result()) + self.generate_gotref(code, handle_null=True) assigned = True if entry.is_cglobal: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: if not self.cf_is_null: if self.cf_maybe_null: - code.put_xdecref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_xdecref_set(code, rhs.result_as(self.ctype())) else: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: assigned = False if is_external_ref: - code.put_giveref(rhs.py_result()) + rhs.generate_giveref(code) if not self.type.is_memoryviewslice: if not assigned: if overloaded_assignment: - result = rhs.result() + result = rhs.move_result_rhs() if exception_check == '+': - translate_cpp_exception(code, self.pos, '%s = %s;' % (self.result(), result), exception_value, self.in_nogil_context) + translate_cpp_exception( + code, self.pos, + '%s = %s;' % (self.result(), result), + self.result() if self.type.is_pyobject else None, + exception_value, self.in_nogil_context) else: code.putln('%s = %s;' % (self.result(), result)) else: - result = rhs.result_as(self.ctype()) - code.putln('%s = %s;' % (self.result(), result)) + result = rhs.move_result_rhs_as(self.ctype()) + + if is_pythran_expr(self.type): + code.putln('new (&%s) decltype(%s){%s};' % (self.result(), self.result(), result)) + elif result != self.result(): + code.putln('%s = %s;' % (self.result(), result)) if debug_disposal_code: print("NameNode.generate_assignment_code:") print("...generating post-assignment code for %s" % rhs) @@ -2267,7 +2586,7 @@ def generate_deletion_code(self, code, ignore_nonexisting=False): if self.entry is None: - return # There was an error earlier + return # There was an error earlier elif self.entry.is_pyclass_attr: namespace = self.entry.scope.namespace_cname interned_cname = code.intern_identifier(self.entry.name) @@ -2303,26 +2622,20 @@ if self.cf_maybe_null and not ignore_nonexisting: code.put_error_if_unbound(self.pos, self.entry) - if self.entry.type.is_pyobject: - if self.entry.in_closure: - # generator - if ignore_nonexisting and self.cf_maybe_null: - code.put_xgotref(self.result()) - else: - code.put_gotref(self.result()) - if ignore_nonexisting and self.cf_maybe_null: - code.put_xdecref(self.result(), self.ctype()) - else: - code.put_decref(self.result(), self.ctype()) - code.putln('%s = NULL;' % self.result()) + if self.entry.in_closure: + # generator + self.generate_gotref(code, handle_null=True, maybe_null_extra_check=ignore_nonexisting) + if ignore_nonexisting and self.cf_maybe_null: + code.put_xdecref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: - code.put_xdecref_memoryviewslice(self.entry.cname, - have_gil=not self.nogil) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: error(self.pos, "Deletion of C names not supported") def annotate(self, code): - if hasattr(self, 'is_called') and self.is_called: + if getattr(self, 'is_called', False): pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1) if self.type.is_pyobject: style, text = 'py_call', 'python function (%s)' @@ -2330,6 +2643,11 @@ style, text = 'c_call', 'c function (%s)' code.annotate(pos, AnnotationItem(style, text % self.type, size=len(self.name))) + def get_known_standard_library_import(self): + if self.entry: + return self.entry.known_standard_library_import + return None + class BackquoteNode(ExprNode): # `expr` # @@ -2356,7 +2674,7 @@ self.result(), self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ImportNode(ExprNode): @@ -2375,44 +2693,61 @@ # relative to the current module. # None: decide the level according to language level and # directives + # get_top_level_module int true: return top-level module, false: return imported module + # module_names TupleNode the separate names of the module and submodules, or None type = py_object_type + module_names = None + get_top_level_module = False + is_temp = True - subexprs = ['module_name', 'name_list'] + subexprs = ['module_name', 'name_list', 'module_names'] def analyse_types(self, env): if self.level is None: - if (env.directives['py2_import'] or - Future.absolute_import not in env.global_scope().context.future_directives): + # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first. + if env.global_scope().parent_module and ( + env.directives['py2_import'] or + Future.absolute_import not in env.global_scope().context.future_directives): self.level = -1 else: self.level = 0 module_name = self.module_name.analyse_types(env) self.module_name = module_name.coerce_to_pyobject(env) + assert self.module_name.is_string_literal if self.name_list: name_list = self.name_list.analyse_types(env) self.name_list = name_list.coerce_to_pyobject(env) - self.is_temp = 1 + elif '.' in self.module_name.value: + self.module_names = TupleNode(self.module_name.pos, args=[ + IdentifierStringNode(self.module_name.pos, value=part, constant_result=part) + for part in map(StringEncoding.EncodedString, self.module_name.value.split('.')) + ]).analyse_types(env) return self gil_message = "Python import" def generate_result_code(self, code): - if self.name_list: - name_list_code = self.name_list.py_result() + assert self.module_name.is_string_literal + module_name = self.module_name.value + + if self.level == 0 and not self.name_list and not self.get_top_level_module: + if self.module_names: + assert self.module_names.is_literal # make sure we create the tuple only once + code.globalstate.use_utility_code(UtilityCode.load_cached("ImportDottedModule", "ImportExport.c")) + import_code = "__Pyx_ImportDottedModule(%s, %s)" % ( + self.module_name.py_result(), + self.module_names.py_result() if self.module_names else 'NULL', + ) else: - name_list_code = "0" + code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) + import_code = "__Pyx_Import(%s, %s, %d)" % ( + self.module_name.py_result(), + self.name_list.py_result() if self.name_list else '0', + self.level) - code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) - import_code = "__Pyx_Import(%s, %s, %d)" % ( - self.module_name.py_result(), - name_list_code, - self.level) - - if (self.level <= 0 and - self.module_name.is_string_literal and - self.module_name.value in utility_code_for_imports): - helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value] + if self.level <= 0 and module_name in utility_code_for_imports: + helper_func, code_name, code_file = utility_code_for_imports[module_name] code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file)) import_code = '%s(%s)' % (helper_func, import_code) @@ -2420,7 +2755,10 @@ self.result(), import_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) + + def get_known_standard_library_import(self): + return self.module_name.value class IteratorNode(ExprNode): @@ -2433,7 +2771,6 @@ type = py_object_type iter_func_ptr = None counter_cname = None - cpp_iterator_cname = None reversed = False # currently only used for list/tuple types (see Optimize.py) is_async = False @@ -2446,7 +2783,7 @@ # C array iteration will be transformed later on self.type = self.sequence.type elif self.sequence.type.is_cpp_class: - self.analyse_cpp_types(env) + return CppIteratorNode(self.pos, sequence=self.sequence).analyse_types(env) else: self.sequence = self.sequence.coerce_to_pyobject(env) if self.sequence.type in (list_type, tuple_type): @@ -2476,65 +2813,10 @@ return sequence_type return py_object_type - def analyse_cpp_types(self, env): - sequence_type = self.sequence.type - if sequence_type.is_ptr: - sequence_type = sequence_type.base_type - begin = sequence_type.scope.lookup("begin") - end = sequence_type.scope.lookup("end") - if (begin is None - or not begin.type.is_cfunction - or begin.type.args): - error(self.pos, "missing begin() on %s" % self.sequence.type) - self.type = error_type - return - if (end is None - or not end.type.is_cfunction - or end.type.args): - error(self.pos, "missing end() on %s" % self.sequence.type) - self.type = error_type - return - iter_type = begin.type.return_type - if iter_type.is_cpp_class: - if env.lookup_operator_for_types( - self.pos, - "!=", - [iter_type, end.type.return_type]) is None: - error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: - error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: - error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - self.type = iter_type - elif iter_type.is_ptr: - if not (iter_type == end.type.return_type): - error(self.pos, "incompatible types for begin() and end()") - self.type = iter_type - else: - error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) - self.type = error_type - return - def generate_result_code(self, code): sequence_type = self.sequence.type if sequence_type.is_cpp_class: - if self.sequence.is_name: - # safe: C++ won't allow you to reassign to class references - begin_func = "%s.begin" % self.sequence.result() - else: - sequence_type = PyrexTypes.c_ptr_type(sequence_type) - self.cpp_iterator_cname = code.funcstate.allocate_temp(sequence_type, manage_ref=False) - code.putln("%s = &%s;" % (self.cpp_iterator_cname, self.sequence.result())) - begin_func = "%s->begin" % self.cpp_iterator_cname - # TODO: Limit scope. - code.putln("%s = %s();" % (self.result(), begin_func)) - return + assert False, "Should have been changed to CppIteratorNode" if sequence_type.is_array or sequence_type.is_ptr: raise InternalError("for in carray slice not transformed") @@ -2576,12 +2858,12 @@ self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) # PyObject_GetIter() fails if "tp_iternext" is not set, but the check below # makes it visible to the C compiler that the pointer really isn't NULL, so that # it can distinguish between the special cases and the generic case - code.putln("%s = Py_TYPE(%s)->tp_iternext; %s" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s); %s" % ( self.iter_func_ptr, self.py_result(), code.error_goto_if_null(self.iter_func_ptr, self.pos))) if self.may_be_a_sequence: @@ -2623,28 +2905,14 @@ self.counter_cname, inc_dec, code.error_goto_if_null(result_name, self.pos))) - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("#endif") def generate_iter_next_result_code(self, result_name, code): sequence_type = self.sequence.type if self.reversed: code.putln("if (%s < 0) break;" % self.counter_cname) - if sequence_type.is_cpp_class: - if self.cpp_iterator_cname: - end_func = "%s->end" % self.cpp_iterator_cname - else: - end_func = "%s.end" % self.sequence.result() - # TODO: Cache end() call? - code.putln("if (!(%s != %s())) break;" % ( - self.result(), - end_func)) - code.putln("%s = *%s;" % ( - result_name, - self.result())) - code.putln("++%s;" % self.result()) - return - elif sequence_type is list_type: + if sequence_type is list_type: self.generate_next_sequence_item('List', result_name, code) return elif sequence_type is tuple_type: @@ -2669,13 +2937,12 @@ code.putln("if (unlikely(!%s)) {" % result_name) code.putln("PyObject* exc_type = PyErr_Occurred();") code.putln("if (exc_type) {") - code.putln("if (likely(exc_type == PyExc_StopIteration ||" - " PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();") + code.putln("if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();") code.putln("else %s" % code.error_goto(self.pos)) code.putln("}") code.putln("break;") code.putln("}") - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("}") def free_temps(self, code): @@ -2684,8 +2951,114 @@ if self.iter_func_ptr: code.funcstate.release_temp(self.iter_func_ptr) self.iter_func_ptr = None - if self.cpp_iterator_cname: - code.funcstate.release_temp(self.cpp_iterator_cname) + ExprNode.free_temps(self, code) + + +class CppIteratorNode(ExprNode): + # Iteration over a C++ container. + # Created at the analyse_types stage by IteratorNode + cpp_sequence_cname = None + cpp_attribute_op = "." + extra_dereference = "" + is_temp = True + + subexprs = ['sequence'] + + def analyse_types(self, env): + sequence_type = self.sequence.type + if sequence_type.is_ptr: + sequence_type = sequence_type.base_type + begin = sequence_type.scope.lookup("begin") + end = sequence_type.scope.lookup("end") + if (begin is None + or not begin.type.is_cfunction + or begin.type.args): + error(self.pos, "missing begin() on %s" % self.sequence.type) + self.type = error_type + return self + if (end is None + or not end.type.is_cfunction + or end.type.args): + error(self.pos, "missing end() on %s" % self.sequence.type) + self.type = error_type + return self + iter_type = begin.type.return_type + if iter_type.is_cpp_class: + if env.directives['cpp_locals']: + self.extra_dereference = "*" + if env.lookup_operator_for_types( + self.pos, + "!=", + [iter_type, end.type.return_type]) is None: + error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: + error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: + error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + self.type = iter_type + elif iter_type.is_ptr: + if not (iter_type == end.type.return_type): + error(self.pos, "incompatible types for begin() and end()") + self.type = iter_type + else: + error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) + self.type = error_type + return self + + def generate_result_code(self, code): + sequence_type = self.sequence.type + # essentially 3 options: + if self.sequence.is_name or self.sequence.is_attribute: + # 1) is a name and can be accessed directly; + # assigning to it may break the container, but that's the responsibility + # of the user + code.putln("%s = %s%sbegin();" % (self.result(), + self.sequence.result(), + self.cpp_attribute_op)) + else: + # (while it'd be nice to limit the scope of the loop temp, it's essentially + # impossible to do while supporting generators) + temp_type = sequence_type + if temp_type.is_reference: + # 2) Sequence is a reference (often obtained by dereferencing a pointer); + # make the temp a pointer so we are not sensitive to users reassigning + # the pointer than it came from + temp_type = PyrexTypes.CPtrType(sequence_type.ref_base_type) + if temp_type.is_ptr or code.globalstate.directives['cpp_locals']: + self.cpp_attribute_op = "->" + # 3) (otherwise) sequence comes from a function call or similar, so we must + # create a temp to store it in + self.cpp_sequence_cname = code.funcstate.allocate_temp(temp_type, manage_ref=False) + code.putln("%s = %s%s;" % (self.cpp_sequence_cname, + "&" if temp_type.is_ptr else "", + self.sequence.move_result_rhs())) + code.putln("%s = %s%sbegin();" % (self.result(), self.cpp_sequence_cname, + self.cpp_attribute_op)) + + def generate_iter_next_result_code(self, result_name, code): + # end call isn't cached to support containers that allow adding while iterating + # (much as this is usually a bad idea) + code.putln("if (!(%s%s != %s%send())) break;" % ( + self.extra_dereference, + self.result(), + self.cpp_sequence_cname or self.sequence.result(), + self.cpp_attribute_op)) + code.putln("%s = *%s%s;" % ( + result_name, + self.extra_dereference, + self.result())) + code.putln("++%s%s;" % (self.extra_dereference, self.result())) + + def free_temps(self, code): + if self.cpp_sequence_cname: + code.funcstate.release_temp(self.cpp_sequence_cname) + # skip over IteratorNode since we don't use any of the temps it does ExprNode.free_temps(self, code) @@ -2717,8 +3090,8 @@ item_type = env.lookup_operator_for_types(self.pos, "*", [iterator_type]).type.return_type if item_type.is_reference: item_type = item_type.ref_base_type - if item_type.is_const: - item_type = item_type.const_base_type + if item_type.is_cv_qualified: + item_type = item_type.cv_base_type return item_type else: # Avoid duplication of complicated logic. @@ -2767,7 +3140,7 @@ self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class AsyncNextNode(AtomicExprNode): @@ -2797,25 +3170,25 @@ self.result(), self.iterator.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class WithExitCallNode(ExprNode): # The __exit__() call of a 'with' statement. Used in both the # except and finally clauses. - # with_stat WithStatNode the surrounding 'with' statement - # args TupleNode or ResultStatNode the exception info tuple - # await AwaitExprNode the await expression of an 'async with' statement + # with_stat WithStatNode the surrounding 'with' statement + # args TupleNode or ResultStatNode the exception info tuple + # await_expr AwaitExprNode the await expression of an 'async with' statement - subexprs = ['args', 'await'] + subexprs = ['args', 'await_expr'] test_if_run = True - await = None + await_expr = None def analyse_types(self, env): self.args = self.args.analyse_types(env) - if self.await: - self.await = self.await.analyse_types(env) + if self.await_expr: + self.await_expr = self.await_expr.analyse_types(env) self.type = PyrexTypes.c_bint_type self.is_temp = True return self @@ -2840,14 +3213,14 @@ self.args.free_temps(code) code.putln(code.error_goto_if_null(result_var, self.pos)) - code.put_gotref(result_var) + code.put_gotref(result_var, py_object_type) - if self.await: + if self.await_expr: # FIXME: result_var temp currently leaks into the closure - self.await.generate_evaluation_code(code, source_cname=result_var, decref_source=True) - code.putln("%s = %s;" % (result_var, self.await.py_result())) - self.await.generate_post_assignment_code(code) - self.await.free_temps(code) + self.await_expr.generate_evaluation_code(code, source_cname=result_var, decref_source=True) + code.putln("%s = %s;" % (result_var, self.await_expr.py_result())) + self.await_expr.generate_post_assignment_code(code) + self.await_expr.free_temps(code) if self.result_is_used: self.allocate_temp_result(code) @@ -2905,7 +3278,7 @@ return self def analyse_target_declaration(self, env): - pass + self.is_target = True def generate_result_code(self, code): pass @@ -2972,6 +3345,7 @@ # type = unicode_type is_temp = True + gil_message = "String concatenation" subexprs = ['values'] @@ -2994,7 +3368,7 @@ list_var, num_items, code.error_goto_if_null(list_var, self.pos))) - code.put_gotref(list_var) + code.put_gotref(list_var, py_object_type) code.putln("%s = 0;" % ulength_var) code.putln("%s = 127;" % max_char_var) # at least ASCII character range @@ -3007,12 +3381,27 @@ is_ascii = False if isinstance(node, UnicodeNode): try: + # most strings will be ASCII or at least Latin-1 node.value.encode('iso8859-1') max_char_value = '255' node.value.encode('us-ascii') is_ascii = True except UnicodeEncodeError: - pass + if max_char_value != '255': + # not ISO8859-1 => check BMP limit + max_char = max(map(ord, node.value)) + if max_char < 0xD800: + # BMP-only, no surrogate pairs used + max_char_value = '65535' + ulength = str(len(node.value)) + elif max_char >= 65536: + # cleary outside of BMP, and not on a 16-bit Unicode system + max_char_value = '1114111' + ulength = str(len(node.value)) + else: + # not really worth implementing a check for surrogate pairs here + # drawback: C code can differ when generating on Py2 with 2-byte Unicode + pass else: ulength = str(len(node.value)) elif isinstance(node, FormattedValueNode) and node.value.type.is_numeric: @@ -3023,7 +3412,7 @@ max_char_var, max_char_value, max_char_var, max_char_value, max_char_var)) code.putln("%s += %s;" % (ulength_var, ulength)) - code.put_giveref(node.py_result()) + node.generate_giveref(code) code.putln('PyTuple_SET_ITEM(%s, %s, %s);' % (list_var, i, node.py_result())) node.generate_post_assignment_code(code) node.free_temps(code) @@ -3038,7 +3427,7 @@ ulength_var, max_char_var, code.error_goto_if_null(self.py_result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.put_decref_clear(list_var, py_object_type) code.funcstate.release_temp(list_var) @@ -3050,7 +3439,7 @@ # {}-delimited portions of an f-string # # value ExprNode The expression itself - # conversion_char str or None Type conversion (!s, !r, !a, or none) + # conversion_char str or None Type conversion (!s, !r, !a, none, or 'd' for integer conversion) # format_spec JoinedStrNode or None Format string passed to __format__ # c_format_spec str or None If not None, formatting can be done at the C level @@ -3059,11 +3448,13 @@ type = unicode_type is_temp = True c_format_spec = None + gil_message = "String formatting" find_conversion_func = { - 's': 'PyObject_Str', + 's': 'PyObject_Unicode', 'r': 'PyObject_Repr', 'a': 'PyObject_ASCII', # NOTE: mapped to PyObject_Repr() in Py2 + 'd': '__Pyx_PyNumber_IntOrLong', # NOTE: internal mapping for '%d' formatting }.get def may_be_none(self): @@ -3081,7 +3472,7 @@ self.format_spec = self.format_spec.analyse_types(env).coerce_to_pyobject(env) if self.c_format_spec is None: self.value = self.value.coerce_to_pyobject(env) - if not self.format_spec and not self.conversion_char: + if not self.format_spec and (not self.conversion_char or self.conversion_char == 's'): if self.value.type is unicode_type and not self.value.may_be_none(): # value is definitely a unicode string and we don't format it any special return self.value @@ -3095,7 +3486,7 @@ self.result(), convert_func_call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) return value_result = self.value.py_result() @@ -3134,7 +3525,7 @@ value_result, format_spec, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) #------------------------------------------------------------------- @@ -3172,7 +3563,7 @@ return self.temp_code -class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): +class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): """ Implements cython.parallel.threadid() """ @@ -3211,7 +3602,7 @@ # in most cases, indexing will return a safe reference to an object in a container, # so we consider the result safe if the base object is return self.base.is_ephemeral() or self.base.type in ( - basestring_type, str_type, bytes_type, unicode_type) + basestring_type, str_type, bytes_type, bytearray_type, unicode_type) def check_const_addr(self): return self.base.check_const_addr() and self.index.check_const() @@ -3248,10 +3639,6 @@ is_subscript = True is_fused_index = False - def __init__(self, pos, index, **kw): - ExprNode.__init__(self, pos, index=index, **kw) - self._index = index - def calculate_constant_result(self): self.constant_result = self.base.constant_result[self.index.constant_result] @@ -3275,7 +3662,7 @@ return False if isinstance(self.index, SliceNode): # slicing! - if base_type in (bytes_type, str_type, unicode_type, + if base_type in (bytes_type, bytearray_type, str_type, unicode_type, basestring_type, list_type, tuple_type): return False return ExprNode.may_be_none(self) @@ -3285,9 +3672,9 @@ def analyse_as_type(self, env): base_type = self.base.analyse_as_type(env) - if base_type and not base_type.is_pyobject: - if base_type.is_cpp_class: - if isinstance(self.index, TupleNode): + if base_type and (not base_type.is_pyobject or base_type.python_type_constructor_name): + if base_type.is_cpp_class or base_type.python_type_constructor_name: + if self.index.is_sequence_constructor: template_values = self.index.args else: template_values = [self.index] @@ -3296,10 +3683,22 @@ positional_args=template_values, keyword_args=None) return type_node.analyse(env, base_type=base_type) + elif self.index.is_slice or self.index.is_sequence_constructor: + # memory view + from . import MemoryView + env.use_utility_code(MemoryView.view_utility_code) + axes = [self.index] if self.index.is_slice else list(self.index.args) + return PyrexTypes.MemoryViewSliceType(base_type, MemoryView.get_axes_specs(env, axes)) else: + # C array index = self.index.compile_time_value(env) if index is not None: - return PyrexTypes.CArrayType(base_type, int(index)) + try: + index = int(index) + except (ValueError, TypeError): + pass + else: + return PyrexTypes.CArrayType(base_type, index) error(self.pos, "Array size must be a compile time constant") return None @@ -3320,6 +3719,8 @@ bytearray_type, list_type, tuple_type): # slicing these returns the same type return base_type + elif base_type.is_memoryviewslice: + return base_type else: # TODO: Handle buffers (hopefully without too much redundancy). return py_object_type @@ -3362,6 +3763,23 @@ index += base_type.size if 0 <= index < base_type.size: return base_type.components[index] + elif base_type.is_memoryviewslice: + if base_type.ndim == 0: + pass # probably an error, but definitely don't know what to do - return pyobject for now + if base_type.ndim == 1: + return base_type.dtype + else: + return PyrexTypes.MemoryViewSliceType(base_type.dtype, base_type.axes[1:]) + + if self.index.is_sequence_constructor and base_type.is_memoryviewslice: + inferred_type = base_type + for a in self.index.args: + if not inferred_type.is_memoryviewslice: + break # something's gone wrong + inferred_type = IndexNode(self.pos, base=ExprNode(self.base.pos, type=inferred_type), + index=a).infer_type(env) + else: + return inferred_type if base_type.is_cpp_class: class FakeOperand: @@ -3375,6 +3793,10 @@ if index_func is not None: return index_func.type.return_type + if is_pythran_expr(base_type) and is_pythran_expr(index_type): + index_with_type = (self.index, index_type) + return PythranExpr(pythran_indexing_type(base_type, [index_with_type])) + # may be slicing or indexing, we don't know if base_type in (unicode_type, str_type): # these types always returns their own type on Python indexing/slicing @@ -3435,6 +3857,8 @@ if not base_type.is_cfunction: self.index = self.index.analyse_types(env) self.original_index_type = self.index.type + if self.original_index_type.is_reference: + self.original_index_type = self.original_index_type.ref_base_type if base_type.is_unicode_char: # we infer Py_UNICODE/Py_UCS4 for unicode strings in some @@ -3466,14 +3890,22 @@ def analyse_as_pyobject(self, env, is_slice, getting, setting): base_type = self.base.type - if self.index.type.is_int and base_type is not dict_type: + if self.index.type.is_unicode_char and base_type is not dict_type: + # TODO: eventually fold into case below and remove warning, once people have adapted their code + warning(self.pos, + "Item lookup of unicode character codes now always converts to a Unicode string. " + "Use an explicit C integer cast to get back the previous integer lookup behaviour.", level=1) + self.index = self.index.coerce_to_pyobject(env) + self.is_temp = 1 + elif self.index.type.is_int and base_type is not dict_type: if (getting + and not env.directives['boundscheck'] and (base_type in (list_type, tuple_type, bytearray_type)) and (not self.index.type.signed or not env.directives['wraparound'] or (isinstance(self.index, IntNode) and self.index.has_constant_result() and self.index.constant_result >= 0)) - and not env.directives['boundscheck']): + ): self.is_temp = 0 else: self.is_temp = 1 @@ -3493,7 +3925,7 @@ else: # not using 'uchar' to enable fast and safe error reporting as '-1' self.type = PyrexTypes.c_int_type - elif is_slice and base_type in (bytes_type, str_type, unicode_type, list_type, tuple_type): + elif is_slice and base_type in (bytes_type, bytearray_type, str_type, unicode_type, list_type, tuple_type): self.type = base_type else: item_type = None @@ -3513,6 +3945,8 @@ def analyse_as_c_array(self, env, is_slice): base_type = self.base.type self.type = base_type.base_type + if self.type.is_cpp_class: + self.type = PyrexTypes.CReferenceType(self.type) if is_slice: self.type = base_type elif self.index.type.is_pyobject: @@ -3537,7 +3971,7 @@ if self.exception_check: if not setting: self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) self.index = self.index.coerce_to(func_type.args[0].type, env) self.type = func_type.return_type @@ -3555,6 +3989,9 @@ if base_type.templates is None: error(self.pos, "Can only parameterize template functions.") self.type = error_type + elif self.type_indices is None: + # Error recorded earlier. + self.type = error_type elif len(base_type.templates) != len(self.type_indices): error(self.pos, "Wrong number of template arguments: expected %s, got %s" % ( (len(base_type.templates), len(self.type_indices)))) @@ -3591,25 +4028,45 @@ else: indices = [self.index] - base_type = self.base.type + base = self.base + base_type = base.type replacement_node = None if base_type.is_memoryviewslice: # memoryviewslice indexing or slicing from . import MemoryView + if base.is_memview_slice: + # For memory views, "view[i][j]" is the same as "view[i, j]" => use the latter for speed. + merged_indices = base.merged_indices(indices) + if merged_indices is not None: + base = base.base + base_type = base.type + indices = merged_indices have_slices, indices, newaxes = MemoryView.unellipsify(indices, base_type.ndim) if have_slices: - replacement_node = MemoryViewSliceNode(self.pos, indices=indices, base=self.base) + replacement_node = MemoryViewSliceNode(self.pos, indices=indices, base=base) else: - replacement_node = MemoryViewIndexNode(self.pos, indices=indices, base=self.base) - - elif base_type.is_buffer and len(indices) == base_type.ndim: - # Buffer indexing - is_buffer_access = True - indices = [index.analyse_types(env) for index in indices] - if all(index.type.is_int for index in indices): - replacement_node = BufferIndexNode(self.pos, indices=indices, base=self.base) - # On cloning, indices is cloned. Otherwise, unpack index into indices. - assert not isinstance(self.index, CloneNode) + replacement_node = MemoryViewIndexNode(self.pos, indices=indices, base=base) + elif base_type.is_buffer or base_type.is_pythran_expr: + if base_type.is_pythran_expr or len(indices) == base_type.ndim: + # Buffer indexing + is_buffer_access = True + indices = [index.analyse_types(env) for index in indices] + if base_type.is_pythran_expr: + do_replacement = all( + index.type.is_int or index.is_slice or index.type.is_pythran_expr + for index in indices) + if do_replacement: + for i,index in enumerate(indices): + if index.is_slice: + index = SliceIntNode(index.pos, start=index.start, stop=index.stop, step=index.step) + index = index.analyse_types(env) + indices[i] = index + else: + do_replacement = all(index.type.is_int for index in indices) + if do_replacement: + replacement_node = BufferIndexNode(self.pos, indices=indices, base=base) + # On cloning, indices is cloned. Otherwise, unpack index into indices. + assert not isinstance(self.index, CloneNode) if replacement_node is not None: replacement_node = replacement_node.analyse_types(env, getting) @@ -3774,6 +4231,8 @@ if not self.is_temp: # all handled in self.calculate_result_code() return + + utility_code = None if self.type.is_pyobject: error_value = 'NULL' if self.index.type.is_int: @@ -3783,32 +4242,38 @@ function = "__Pyx_GetItemInt_Tuple" else: function = "__Pyx_GetItemInt" - code.globalstate.use_utility_code( - TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c")) + utility_code = TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c") else: if self.base.type is dict_type: function = "__Pyx_PyDict_GetItem" - code.globalstate.use_utility_code( - UtilityCode.load_cached("DictGetItem", "ObjectHandling.c")) + utility_code = UtilityCode.load_cached("DictGetItem", "ObjectHandling.c") + elif self.base.type is py_object_type and self.index.type in (str_type, unicode_type): + # obj[str] is probably doing a dict lookup + function = "__Pyx_PyObject_Dict_GetItem" + utility_code = UtilityCode.load_cached("DictGetItem", "ObjectHandling.c") else: - function = "PyObject_GetItem" + function = "__Pyx_PyObject_GetItem" + code.globalstate.use_utility_code( + TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c")) + utility_code = UtilityCode.load_cached("ObjectGetItem", "ObjectHandling.c") elif self.type.is_unicode_char and self.base.type is unicode_type: assert self.index.type.is_int function = "__Pyx_GetItemInt_Unicode" error_value = '(Py_UCS4)-1' - code.globalstate.use_utility_code( - UtilityCode.load_cached("GetItemIntUnicode", "StringTools.c")) + utility_code = UtilityCode.load_cached("GetItemIntUnicode", "StringTools.c") elif self.base.type is bytearray_type: assert self.index.type.is_int assert self.type.is_int function = "__Pyx_GetItemInt_ByteArray" error_value = '-1' - code.globalstate.use_utility_code( - UtilityCode.load_cached("GetItemIntByteArray", "StringTools.c")) + utility_code = UtilityCode.load_cached("GetItemIntByteArray", "StringTools.c") elif not (self.base.type.is_cpp_class and self.exception_check): assert False, "unexpected type %s and base type %s for indexing" % ( self.type, self.base.type) + if utility_code is not None: + code.globalstate.use_utility_code(utility_code) + if self.index.type.is_int: index_code = self.index.result() else: @@ -3818,6 +4283,7 @@ translate_cpp_exception(code, self.pos, "%s = %s[%s];" % (self.result(), self.base.result(), self.index.result()), + self.result() if self.type.is_pyobject else None, self.exception_value, self.in_nogil_context) else: error_check = '!%s' if error_value == 'NULL' else '%%s == %s' % error_value @@ -3830,7 +4296,7 @@ self.extra_index_params(code), code.error_goto_if(error_check % self.result(), self.pos))) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_setitem_code(self, value_code, code): if self.index.type.is_int: @@ -3866,7 +4332,7 @@ self.pos)) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: @@ -3875,8 +4341,7 @@ value_code = self._check_byte_value(code, rhs) self.generate_setitem_code(value_code, code) elif self.base.type.is_cpp_class and self.exception_check and self.exception_check == '+': - if overloaded_assignment and exception_check and \ - self.exception_value != exception_value: + if overloaded_assignment and exception_check and self.exception_value != exception_value: # Handle the case that both the index operator and the assignment # operator have a c++ exception handler and they are not the same. translate_double_cpp_exception(code, self.pos, self.type, @@ -3888,6 +4353,7 @@ # both exception handlers are the same. translate_cpp_exception(code, self.pos, "%s = %s;" % (self.result(), rhs.result()), + self.result() if self.type.is_pyobject else None, self.exception_value, self.in_nogil_context) else: code.putln( @@ -3976,6 +4442,9 @@ # Whether we're assigning to a buffer (in that case it needs to be writable) writable_needed = False + # Any indexing temp variables that we need to clean up. + index_temps = () + def analyse_target_types(self, env): self.analyse_types(env, getting=False) @@ -3985,7 +4454,7 @@ indexing and slicing subclasses """ # self.indices are already analyzed - if not self.base.is_name: + if not self.base.is_name and not is_pythran_expr(self.base.type): error(self.pos, "Can only index buffer variables") self.type = error_type return self @@ -4004,11 +4473,15 @@ return self def analyse_buffer_index(self, env, getting): - self.base = self.base.coerce_to_simple(env) - self.type = self.base.type.dtype + if is_pythran_expr(self.base.type): + index_with_type_list = [(idx, idx.type) for idx in self.indices] + self.type = PythranExpr(pythran_indexing_type(self.base.type, index_with_type_list)) + else: + self.base = self.base.coerce_to_simple(env) + self.type = self.base.type.dtype self.buffer_type = self.base.type - if getting and self.type.is_pyobject: + if getting and (self.type.is_pyobject or self.type.is_pythran_expr): self.is_temp = True def analyse_assignment(self, rhs): @@ -4024,10 +4497,6 @@ def nogil_check(self, env): if self.is_buffer_access or self.is_memview_index: - if env.directives['boundscheck']: - warning(self.pos, "Use boundscheck(False) for faster access", - level=1) - if self.type.is_pyobject: error(self.pos, "Cannot access buffer with object dtype without gil") self.type = error_type @@ -4041,20 +4510,26 @@ base = base.arg return base.type.get_entry(base) + def get_index_in_temp(self, code, ivar): + ret = code.funcstate.allocate_temp( + PyrexTypes.widest_numeric_type( + ivar.type, + PyrexTypes.c_ssize_t_type if ivar.type.signed else PyrexTypes.c_size_t_type), + manage_ref=False) + code.putln("%s = %s;" % (ret, ivar.result())) + return ret + def buffer_lookup_code(self, code): """ ndarray[1, 2, 3] and memslice[1, 2, 3] """ - # Assign indices to temps of at least (s)size_t to allow further index calculations. - index_temps = [ - code.funcstate.allocate_temp( - PyrexTypes.widest_numeric_type( - ivar.type, PyrexTypes.c_ssize_t_type if ivar.type.signed else PyrexTypes.c_size_t_type), - manage_ref=False) - for ivar in self.indices] + if self.in_nogil_context: + if self.is_buffer_access or self.is_memview_index: + if code.globalstate.directives['boundscheck']: + warning(self.pos, "Use boundscheck(False) for faster access", level=1) - for temp, index in zip(index_temps, self.indices): - code.putln("%s = %s;" % (temp, index.result())) + # Assign indices to temps of at least (s)size_t to allow further index calculations. + self.index_temps = index_temps = [self.get_index_in_temp(code,ivar) for ivar in self.indices] # Generate buffer access code using these temps from . import Buffer @@ -4082,6 +4557,27 @@ rhs.free_temps(code) def generate_buffer_setitem_code(self, rhs, code, op=""): + base_type = self.base.type + if is_pythran_expr(base_type) and is_pythran_supported_type(rhs.type): + obj = code.funcstate.allocate_temp(PythranExpr(pythran_type(self.base.type)), manage_ref=False) + # We have got to do this because we have to declare pythran objects + # at the beginning of the functions. + # Indeed, Cython uses "goto" statement for error management, and + # RAII doesn't work with that kind of construction. + # Moreover, the way Pythran expressions are made is that they don't + # support move-assignation easily. + # This, we explicitly destroy then in-place new objects in this + # case. + code.putln("__Pyx_call_destructor(%s);" % obj) + code.putln("new (&%s) decltype(%s){%s};" % (obj, obj, self.base.pythran_result())) + code.putln("%s%s %s= %s;" % ( + obj, + pythran_indexing_code(self.indices), + op, + rhs.pythran_result())) + code.funcstate.release_temp(obj) + return + # Used from generate_assignment_code and InPlaceAssignmentNode buffer_entry, ptrexpr = self.buffer_lookup_code(code) @@ -4092,17 +4588,26 @@ manage_ref=False) rhs_code = rhs.result() code.putln("%s = %s;" % (ptr, ptrexpr)) - code.put_gotref("*%s" % ptr) + code.put_gotref("*%s" % ptr, self.buffer_type.dtype) code.putln("__Pyx_INCREF(%s); __Pyx_DECREF(*%s);" % ( rhs_code, ptr)) code.putln("*%s %s= %s;" % (ptr, op, rhs_code)) - code.put_giveref("*%s" % ptr) + code.put_giveref("*%s" % ptr, self.buffer_type.dtype) code.funcstate.release_temp(ptr) else: # Simple case code.putln("*%s %s= %s;" % (ptrexpr, op, rhs.result())) def generate_result_code(self, code): + if is_pythran_expr(self.base.type): + res = self.result() + code.putln("__Pyx_call_destructor(%s);" % res) + code.putln("new (&%s) decltype(%s){%s%s};" % ( + res, + res, + self.base.pythran_result(), + pythran_indexing_code(self.indices))) + return buffer_entry, self.buffer_ptr_code = self.buffer_lookup_code(code) if self.type.is_pyobject: # is_temp is True, so must pull out value and incref it. @@ -4111,6 +4616,12 @@ code.putln("%s = (PyObject *) *%s;" % (self.result(), self.buffer_ptr_code)) code.putln("__Pyx_INCREF((PyObject*)%s);" % self.result()) + def free_subexpr_temps(self, code): + for temp in self.index_temps: + code.funcstate.release_temp(temp) + self.index_temps = () + super(BufferIndexNode, self).free_subexpr_temps(code) + class MemoryViewIndexNode(BufferIndexNode): @@ -4122,9 +4633,15 @@ # memoryviewslice indexing or slicing from . import MemoryView + self.is_pythran_mode = has_np_pythran(env) indices = self.indices have_slices, indices, newaxes = MemoryView.unellipsify(indices, self.base.type.ndim) + if not getting: + self.writable_needed = True + if self.base.is_name or self.base.is_attribute: + self.base.entry.type.writable_needed = True + self.memslice_index = (not newaxes and len(indices) == self.base.type.ndim) axes = [] @@ -4272,6 +4789,37 @@ else: return MemoryCopySlice(self.pos, self) + def merged_indices(self, indices): + """Return a new list of indices/slices with 'indices' merged into the current ones + according to slicing rules. + Is used to implement "view[i][j]" => "view[i, j]". + Return None if the indices cannot (easily) be merged at compile time. + """ + if not indices: + return None + # NOTE: Need to evaluate "self.original_indices" here as they might differ from "self.indices". + new_indices = self.original_indices[:] + indices = indices[:] + for i, s in enumerate(self.original_indices): + if s.is_slice: + if s.start.is_none and s.stop.is_none and s.step.is_none: + # Full slice found, replace by index. + new_indices[i] = indices[0] + indices.pop(0) + if not indices: + return new_indices + else: + # Found something non-trivial, e.g. a partial slice. + return None + elif not s.type.is_int: + # Not a slice, not an integer index => could be anything... + return None + if indices: + if len(new_indices) + len(indices) > self.base.type.ndim: + return None + new_indices += indices + return new_indices + def is_simple(self): if self.is_ellipsis_noop: # TODO: fix SimpleCallNode.is_simple() @@ -4307,7 +4855,7 @@ assert not list(it) buffer_entry.generate_buffer_slice_code( - code, self.original_indices, self.result(), + code, self.original_indices, self.result(), self.type, have_gil=have_gil, have_slices=have_slices, directives=code.globalstate.directives) @@ -4349,6 +4897,7 @@ self.dst.generate_evaluation_code(code) self._generate_assignment_code(rhs, code) self.dst.generate_disposal_code(code) + self.dst.free_temps(code) rhs.generate_disposal_code(code) rhs.free_temps(code) @@ -4409,8 +4958,17 @@ code.putln("%s __pyx_temp_slice = %s;" % (slice_decl, self.dst.result())) dst_temp = "__pyx_temp_slice" + force_strided = False + indices = self.dst.original_indices + for idx in indices: + if isinstance(idx, SliceNode) and not (idx.start.is_none and + idx.stop.is_none and + idx.step.is_none): + force_strided = True + slice_iter_obj = MemoryView.slice_iter(self.dst.type, dst_temp, - self.dst.type.ndim, code) + self.dst.type.ndim, code, + force_strided=force_strided) p = slice_iter_obj.start_loops() if dtype.is_pyobject: @@ -4443,7 +5001,7 @@ return bytes_type elif base_type.is_pyunicode_ptr: return unicode_type - elif base_type in (bytes_type, str_type, unicode_type, + elif base_type in (bytes_type, bytearray_type, str_type, unicode_type, basestring_type, list_type, tuple_type): return base_type elif base_type.is_ptr or base_type.is_array: @@ -4508,13 +5066,13 @@ def analyse_types(self, env, getting=True): self.base = self.base.analyse_types(env) - if self.base.type.is_memoryviewslice: + if self.base.type.is_buffer or self.base.type.is_pythran_expr or self.base.type.is_memoryviewslice: none_node = NoneNode(self.pos) index = SliceNode(self.pos, start=self.start or none_node, stop=self.stop or none_node, step=none_node) - index_node = IndexNode(self.pos, index, base=self.base) + index_node = IndexNode(self.pos, index=index, base=self.base) return index_node.analyse_base_and_index_types( env, getting=getting, setting=not getting, analyse_base=False) @@ -4566,13 +5124,59 @@ ).analyse_types(env) else: c_int = PyrexTypes.c_py_ssize_t_type + + def allow_none(node, default_value, env): + # Coerce to Py_ssize_t, but allow None as meaning the default slice bound. + from .UtilNodes import EvalWithTempExprNode, ResultRefNode + + node_ref = ResultRefNode(node) + new_expr = CondExprNode( + node.pos, + true_val=IntNode( + node.pos, + type=c_int, + value=default_value, + constant_result=int(default_value) if default_value.isdigit() else not_a_constant, + ), + false_val=node_ref.coerce_to(c_int, env), + test=PrimaryCmpNode( + node.pos, + operand1=node_ref, + operator='is', + operand2=NoneNode(node.pos), + ).analyse_types(env) + ).analyse_result_type(env) + return EvalWithTempExprNode(node_ref, new_expr) + if self.start: + if self.start.type.is_pyobject: + self.start = allow_none(self.start, '0', env) self.start = self.start.coerce_to(c_int, env) if self.stop: + if self.stop.type.is_pyobject: + self.stop = allow_none(self.stop, 'PY_SSIZE_T_MAX', env) self.stop = self.stop.coerce_to(c_int, env) self.is_temp = 1 return self + def analyse_as_type(self, env): + base_type = self.base.analyse_as_type(env) + if base_type and not base_type.is_pyobject: + if not self.start and not self.stop: + # memory view + from . import MemoryView + env.use_utility_code(MemoryView.view_utility_code) + none_node = NoneNode(self.pos) + slice_node = SliceNode( + self.pos, + start=none_node, + stop=none_node, + step=none_node, + ) + return PyrexTypes.MemoryViewSliceType( + base_type, MemoryView.get_axes_specs(env, [slice_node])) + return None + nogil_check = Node.gil_error gil_message = "Slicing Python object" @@ -4697,15 +5301,14 @@ start_code, stop_code, code.error_goto_if_null(result, self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: code.globalstate.use_utility_code(self.set_slice_utility_code) - (has_c_start, has_c_stop, c_start, c_stop, - py_start, py_stop, py_slice) = self.get_slice_config() + has_c_start, has_c_stop, c_start, c_stop, py_start, py_stop, py_slice = self.get_slice_config() code.put_error_if_neg(self.pos, "__Pyx_PyObject_SetSlice(%s, %s, %s, %s, %s, %s, %s, %d, %d, %d)" % ( self.base.py_result(), @@ -4918,8 +5521,11 @@ def generate_result_code(self, code): if self.is_literal: - self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2) - code = code.get_cached_constants_writer() + dedup_key = make_dedup_key(self.type, (self,)) + self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2, dedup_key=dedup_key) + code = code.get_cached_constants_writer(self.result_code) + if code is None: + return # already initialised code.mark_pos(self.pos) code.putln( @@ -4929,9 +5535,63 @@ self.stop.py_result(), self.step.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.is_literal: - code.put_giveref(self.py_result()) + self.generate_giveref(code) + +class SliceIntNode(SliceNode): + # start:stop:step in subscript list + # This is just a node to hold start,stop and step nodes that can be + # converted to integers. This does not generate a slice python object. + # + # start ExprNode + # stop ExprNode + # step ExprNode + + is_temp = 0 + + def calculate_constant_result(self): + self.constant_result = slice( + self.start.constant_result, + self.stop.constant_result, + self.step.constant_result) + + def compile_time_value(self, denv): + start = self.start.compile_time_value(denv) + stop = self.stop.compile_time_value(denv) + step = self.step.compile_time_value(denv) + try: + return slice(start, stop, step) + except Exception as e: + self.compile_time_value_error(e) + + def may_be_none(self): + return False + + def analyse_types(self, env): + self.start = self.start.analyse_types(env) + self.stop = self.stop.analyse_types(env) + self.step = self.step.analyse_types(env) + + if not self.start.is_none: + self.start = self.start.coerce_to_integer(env) + if not self.stop.is_none: + self.stop = self.stop.coerce_to_integer(env) + if not self.step.is_none: + self.step = self.step.coerce_to_integer(env) + + if self.start.is_literal and self.stop.is_literal and self.step.is_literal: + self.is_literal = True + self.is_temp = False + return self + + def calculate_result_code(self): + pass + + def generate_result_code(self, code): + for a in self.start,self.stop,self.step: + if isinstance(a, CloneNode): + a.arg.result() class CallNode(ExprNode): @@ -4940,6 +5600,7 @@ may_return_none = None def infer_type(self, env): + # TODO(robertwb): Reduce redundancy with analyse_types. function = self.function func_type = function.infer_type(env) if isinstance(function, NewExprNode): @@ -4953,6 +5614,15 @@ if func_type.is_ptr: func_type = func_type.base_type if func_type.is_cfunction: + if getattr(self.function, 'entry', None) and hasattr(self, 'args'): + alternatives = self.function.entry.all_alternatives() + arg_types = [arg.infer_type(env) for arg in self.args] + func_entry = PyrexTypes.best_match(arg_types, alternatives) + if func_entry: + func_type = func_entry.type + if func_type.is_ptr: + func_type = func_type.base_type + return func_type.return_type return func_type.return_type elif func_type is type_type: if function.is_name and function.entry and function.entry.type: @@ -4964,6 +5634,9 @@ return PyrexTypes.c_double_type elif function.entry.name in Builtin.types_that_construct_their_instance: return result_type + func_type = self.function.analyse_as_type(env) + if func_type and (func_type.is_struct_or_union or func_type.is_cpp_class): + return func_type return py_object_type def type_dependencies(self, env): @@ -4991,6 +5664,32 @@ return False return ExprNode.may_be_none(self) + def set_py_result_type(self, function, func_type=None): + if func_type is None: + func_type = function.type + if func_type is Builtin.type_type and ( + function.is_name and + function.entry and + function.entry.is_builtin and + function.entry.name in Builtin.types_that_construct_their_instance): + # calling a builtin type that returns a specific object type + if function.entry.name == 'float': + # the following will come true later on in a transform + self.type = PyrexTypes.c_double_type + self.result_ctype = PyrexTypes.c_double_type + else: + self.type = Builtin.builtin_types[function.entry.name] + self.result_ctype = py_object_type + self.may_return_none = False + elif function.is_name and function.type_entry: + # We are calling an extension type constructor. As long as we do not + # support __new__(), the result type is clear + self.type = function.type_entry.type + self.result_ctype = py_object_type + self.may_return_none = False + else: + self.type = py_object_type + def analyse_as_type_constructor(self, env): type = self.function.analyse_as_type(env) if type and type.is_struct_or_union: @@ -5008,6 +5707,10 @@ elif type and type.is_cpp_class: self.args = [ arg.analyse_types(env) for arg in self.args ] constructor = type.scope.lookup("") + if not constructor: + error(self.function.pos, "no constructor found for C++ type '%s'" % self.function.name) + self.type = error_type + return self self.function = RawCNameExprNode(self.function.pos, constructor.type) self.function.entry = constructor self.function.set_cname(type.empty_declaration_code()) @@ -5022,7 +5725,7 @@ func_type = self.function_type() if func_type.is_pyobject: self.gil_error() - elif not getattr(func_type, 'nogil', False): + elif not func_type.is_error and not getattr(func_type, 'nogil', False): self.gil_error() gil_message = "Calling gil-requiring function" @@ -5049,6 +5752,7 @@ has_optional_args = False nogil = False analysed = False + overflowcheck = False def compile_time_value(self, denv): function = self.function.compile_time_value(denv) @@ -5058,6 +5762,16 @@ except Exception as e: self.compile_time_value_error(e) + @classmethod + def for_cproperty(cls, pos, obj, entry): + # Create a call node for C property access. + property_scope = entry.scope + getter_entry = property_scope.lookup_here(entry.name) + assert getter_entry, "Getter not found in scope %s: %s" % (property_scope, property_scope.entries) + function = NameNode(pos, name=entry.name, entry=getter_entry, type=getter_entry.type) + node = cls(pos, function=function, args=[obj]) + return node + def analyse_as_type(self, env): attr = self.function.as_cython_attribute() if attr == 'pointer': @@ -5069,6 +5783,11 @@ error(self.args[0].pos, "Unknown type") else: return PyrexTypes.CPtrType(type) + elif attr == 'typeof': + if len(self.args) != 1: + error(self.args.pos, "only one type allowed.") + operand = self.args[0].analyse_types(env) + return operand.type def explicit_args_kwds(self): return self.args, None @@ -5090,38 +5809,35 @@ function.obj = CloneNode(self.self) func_type = self.function_type() - if func_type.is_pyobject: + self.is_numpy_call_with_exprs = False + if (has_np_pythran(env) and function.is_numpy_attribute and + pythran_is_numpy_func_supported(function)): + has_pythran_args = True + self.arg_tuple = TupleNode(self.pos, args = self.args) + self.arg_tuple = self.arg_tuple.analyse_types(env) + for arg in self.arg_tuple.args: + has_pythran_args &= is_pythran_supported_node_or_none(arg) + self.is_numpy_call_with_exprs = bool(has_pythran_args) + if self.is_numpy_call_with_exprs: + env.add_include_file(pythran_get_func_include_file(function)) + return NumPyMethodCallNode.from_node( + self, + function_cname=pythran_functor(function), + arg_tuple=self.arg_tuple, + type=PythranExpr(pythran_func_type(function, self.arg_tuple.args)), + ) + elif func_type.is_pyobject: self.arg_tuple = TupleNode(self.pos, args = self.args) self.arg_tuple = self.arg_tuple.analyse_types(env).coerce_to_pyobject(env) self.args = None - if func_type is Builtin.type_type and function.is_name and \ - function.entry and \ - function.entry.is_builtin and \ - function.entry.name in Builtin.types_that_construct_their_instance: - # calling a builtin type that returns a specific object type - if function.entry.name == 'float': - # the following will come true later on in a transform - self.type = PyrexTypes.c_double_type - self.result_ctype = PyrexTypes.c_double_type - else: - self.type = Builtin.builtin_types[function.entry.name] - self.result_ctype = py_object_type - self.may_return_none = False - elif function.is_name and function.type_entry: - # We are calling an extension type constructor. As - # long as we do not support __new__(), the result type - # is clear - self.type = function.type_entry.type - self.result_ctype = py_object_type - self.may_return_none = False - else: - self.type = py_object_type + self.set_py_result_type(function, func_type) self.is_temp = 1 else: self.args = [ arg.analyse_types(env) for arg in self.args ] self.analyse_c_function_call(env) if func_type.exception_check == '+': self.is_temp = True + return self def function_type(self): @@ -5173,7 +5889,8 @@ else: alternatives = overloaded_entry.all_alternatives() - entry = PyrexTypes.best_match(args, alternatives, self.pos, env) + entry = PyrexTypes.best_match([arg.type for arg in args], + alternatives, self.pos, env, args) if not entry: self.type = PyrexTypes.error_type @@ -5209,7 +5926,7 @@ if formal_arg.not_none: if self.self: self.self = self.self.as_none_safe_node( - "'NoneType' object has no attribute '%s'", + "'NoneType' object has no attribute '%{0}s'".format('.30' if len(entry.name) <= 30 else ''), error='PyExc_AttributeError', format_args=[entry.name]) else: @@ -5235,8 +5952,6 @@ for i in range(min(max_nargs, actual_nargs)): formal_arg = func_type.args[i] formal_type = formal_arg.type - if formal_type.is_const: - formal_type = formal_type.const_base_type arg = args[i].coerce_to(formal_type, env) if formal_arg.not_none: # C methods must do the None checks at *call* time @@ -5258,7 +5973,7 @@ # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) args[i] = arg @@ -5287,7 +6002,7 @@ # case) for i in range(actual_nargs-1): if i == 0 and self.self is not None: - continue # self is ok + continue # self is ok arg = args[i] if arg.nonlocally_immutable(): # locals, C functions, unassignable types are safe. @@ -5303,7 +6018,7 @@ else: #self.args[i] = arg.coerce_to_temp(env) # instead: issue a warning - if i > 0 or i == 1 and self.self is not None: # skip first arg + if i > 0 or i == 1 and self.self is not None: # skip first arg warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0) break @@ -5334,15 +6049,13 @@ # Called in 'nogil' context? self.nogil = env.nogil - if (self.nogil and - func_type.exception_check and - func_type.exception_check != '+'): - env.use_utility_code(pyerr_occurred_withgil_utility_code) # C++ exception handler if func_type.exception_check == '+': - if func_type.exception_value is None: + if needs_cpp_exception_conversion(func_type): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) + self.overflowcheck = env.directives['overflowcheck'] + def calculate_result_code(self): return self.c_call_code() @@ -5357,8 +6070,8 @@ expected_nargs = max_nargs - func_type.optional_arg_count actual_nargs = len(self.args) for formal_arg, actual_arg in args[:expected_nargs]: - arg_code = actual_arg.result_as(formal_arg.type) - arg_list_code.append(arg_code) + arg_code = actual_arg.move_result_rhs_as(formal_arg.type) + arg_list_code.append(arg_code) if func_type.is_overridable: arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod))) @@ -5371,7 +6084,7 @@ arg_list_code.append(optional_args) for actual_arg in self.args[len(formal_args):]: - arg_list_code.append(actual_arg.result()) + arg_list_code.append(actual_arg.move_result_rhs()) result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code)) return result @@ -5382,30 +6095,76 @@ return False # skip allocation of unused result temp return True + def generate_evaluation_code(self, code): + function = self.function + if function.is_name or function.is_attribute: + code.globalstate.use_entry_utility_code(function.entry) + + abs_function_cnames = ('abs', 'labs', '__Pyx_abs_longlong') + is_signed_int = self.type.is_int and self.type.signed + if self.overflowcheck and is_signed_int and function.result() in abs_function_cnames: + code.globalstate.use_utility_code(UtilityCode.load_cached("Common", "Overflow.c")) + code.putln('if (unlikely(%s == __PYX_MIN(%s))) {\ + PyErr_SetString(PyExc_OverflowError,\ + "Trying to take the absolute value of the most negative integer is not defined."); %s; }' % ( + self.args[0].result(), + self.args[0].type.empty_declaration_code(), + code.error_goto(self.pos))) + + if not function.type.is_pyobject or len(self.arg_tuple.args) > 1 or ( + self.arg_tuple.args and self.arg_tuple.is_literal): + super(SimpleCallNode, self).generate_evaluation_code(code) + return + + # Special case 0-args and try to avoid explicit tuple creation for Python calls with 1 arg. + arg = self.arg_tuple.args[0] if self.arg_tuple.args else None + subexprs = (self.self, self.coerced_self, function, arg) + for subexpr in subexprs: + if subexpr is not None: + subexpr.generate_evaluation_code(code) + + code.mark_pos(self.pos) + assert self.is_temp + self.allocate_temp_result(code) + + if arg is None: + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallNoArg", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_CallNoArg(%s); %s" % ( + self.result(), + function.py_result(), + code.error_goto_if_null(self.result(), self.pos))) + else: + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_CallOneArg(%s, %s); %s" % ( + self.result(), + function.py_result(), + arg.py_result(), + code.error_goto_if_null(self.result(), self.pos))) + + self.generate_gotref(code) + + for subexpr in subexprs: + if subexpr is not None: + subexpr.generate_disposal_code(code) + subexpr.free_temps(code) + def generate_result_code(self, code): func_type = self.function_type() - if self.function.is_name or self.function.is_attribute: - code.globalstate.use_entry_utility_code(self.function.entry) if func_type.is_pyobject: - if func_type is not type_type and not self.arg_tuple.args and self.arg_tuple.is_literal: - code.globalstate.use_utility_code(UtilityCode.load_cached( - "PyObjectCallNoArg", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_CallNoArg(%s); %s" % ( - self.result(), - self.function.py_result(), - code.error_goto_if_null(self.result(), self.pos))) - else: - arg_code = self.arg_tuple.py_result() - code.globalstate.use_utility_code(UtilityCode.load_cached( - "PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - self.function.py_result(), - arg_code, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + arg_code = self.arg_tuple.py_result() + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCall", "ObjectHandling.c")) + code.putln( + "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( + self.result(), + self.function.py_result(), + arg_code, + code.error_goto_if_null(self.result(), self.pos))) + self.generate_gotref(code) elif func_type.is_cfunction: if self.has_optional_args: actual_nargs = len(self.args) @@ -5428,13 +6187,15 @@ elif self.type.is_memoryviewslice: assert self.is_temp exc_checks.append(self.type.error_condition(self.result())) - else: + elif func_type.exception_check != '+': exc_val = func_type.exception_value exc_check = func_type.exception_check if exc_val is not None: - exc_checks.append("%s == %s" % (self.result(), exc_val)) + exc_checks.append("%s == %s" % (self.result(), func_type.return_type.cast_code(exc_val))) if exc_check: if self.nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) exc_checks.append("__Pyx_ErrOccurredWithGIL()") else: exc_checks.append("PyErr_Occurred()") @@ -5451,6 +6212,7 @@ lhs = "" if func_type.exception_check == '+': translate_cpp_exception(code, self.pos, '%s%s;' % (lhs, rhs), + self.result() if self.type.is_pyobject else None, func_type.exception_value, self.nogil) else: if exc_checks: @@ -5459,11 +6221,39 @@ goto_error = "" code.putln("%s%s; %s" % (lhs, rhs, goto_error)) if self.type.is_pyobject and self.result(): - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.has_optional_args: code.funcstate.release_temp(self.opt_arg_struct) +class NumPyMethodCallNode(ExprNode): + # Pythran call to a NumPy function or method. + # + # function_cname string the function/method to call + # arg_tuple TupleNode the arguments as an args tuple + + subexprs = ['arg_tuple'] + is_temp = True + may_return_none = True + + def generate_evaluation_code(self, code): + code.mark_pos(self.pos) + self.allocate_temp_result(code) + + assert self.arg_tuple.mult_factor is None + args = self.arg_tuple.args + for arg in args: + arg.generate_evaluation_code(code) + + code.putln("// function evaluation code for numpy function") + code.putln("__Pyx_call_destructor(%s);" % self.result()) + code.putln("new (&%s) decltype(%s){%s{}(%s)};" % ( + self.result(), + self.result(), + self.function_cname, + ", ".join(a.pythran_result() for a in args))) + + class PyMethodCallNode(SimpleCallNode): # Specialised call to a (potential) PyMethodObject with non-constant argument tuple. # Allows the self argument to be injected directly instead of repacking a tuple for it. @@ -5497,10 +6287,8 @@ self_arg = code.funcstate.allocate_temp(py_object_type, manage_ref=True) code.putln("%s = NULL;" % self_arg) - arg_offset_cname = None - if len(args) > 1: - arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) - code.putln("%s = 0;" % arg_offset_cname) + arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) + code.putln("%s = 0;" % arg_offset_cname) def attribute_is_likely_method(attr): obj = attr.obj @@ -5514,7 +6302,7 @@ # not an attribute itself, but might have been assigned from one (e.g. bound method) for assignment in self.function.cf_state: value = assignment.rhs - if value and value.is_attribute and value.obj.type.is_pyobject: + if value and value.is_attribute and value.obj.type and value.obj.type.is_pyobject: if attribute_is_likely_method(value): likely_method = 'likely' break @@ -5532,122 +6320,35 @@ code.put_incref(self_arg, py_object_type) code.put_incref("function", py_object_type) # free method object as early to possible to enable reuse from CPython's freelist - code.put_decref_set(function, "function") - if len(args) > 1: - code.putln("%s = 1;" % arg_offset_cname) + code.put_decref_set(function, py_object_type, "function") + code.putln("%s = 1;" % arg_offset_cname) code.putln("}") code.putln("}") - if not args: - # fastest special case: try to avoid tuple creation - code.putln("if (%s) {" % self_arg) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_CallOneArg(%s, %s); %s" % ( - self.result(), - function, self_arg, - code.error_goto_if_null(self.result(), self.pos))) - code.put_decref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - code.putln("} else {") - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallNoArg", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_CallNoArg(%s); %s" % ( - self.result(), - function, - code.error_goto_if_null(self.result(), self.pos))) - code.putln("}") - code.put_gotref(self.py_result()) - else: - if len(args) == 1: - code.putln("if (!%s) {" % self_arg) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - arg = args[0] - code.putln( - "%s = __Pyx_PyObject_CallOneArg(%s, %s); %s" % ( - self.result(), - function, arg.py_result(), - code.error_goto_if_null(self.result(), self.pos))) - arg.generate_disposal_code(code) - code.put_gotref(self.py_result()) - code.putln("} else {") - arg_offset = 1 - else: - arg_offset = arg_offset_cname - - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyFunctionFastCall", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyCFunctionFastCall", "ObjectHandling.c")) - for test_func, call_prefix in [('PyFunction_Check', 'Py'), ('__Pyx_PyFastCFunction_Check', 'PyC')]: - code.putln("#if CYTHON_FAST_%sCALL" % call_prefix.upper()) - code.putln("if (%s(%s)) {" % (test_func, function)) - code.putln("PyObject *%s[%d] = {%s, %s};" % ( - Naming.quick_temp_cname, - len(args)+1, - self_arg, - ', '.join(arg.py_result() for arg in args))) - code.putln("%s = __Pyx_%sFunction_FastCall(%s, %s+1-%s, %d+%s); %s" % ( - self.result(), - call_prefix, - function, - Naming.quick_temp_cname, - arg_offset, - len(args), - arg_offset, - code.error_goto_if_null(self.result(), self.pos))) - code.put_xdecref_clear(self_arg, py_object_type) - code.put_gotref(self.py_result()) - for arg in args: - arg.generate_disposal_code(code) - code.putln("} else") - code.putln("#endif") - - code.putln("{") - args_tuple = code.funcstate.allocate_temp(py_object_type, manage_ref=True) - code.putln("%s = PyTuple_New(%d+%s); %s" % ( - args_tuple, len(args), arg_offset, - code.error_goto_if_null(args_tuple, self.pos))) - code.put_gotref(args_tuple) - - if len(args) > 1: - code.putln("if (%s) {" % self_arg) - code.putln("__Pyx_GIVEREF(%s); PyTuple_SET_ITEM(%s, 0, %s); %s = NULL;" % ( - self_arg, args_tuple, self_arg, self_arg)) # stealing owned ref in this case - code.funcstate.release_temp(self_arg) - if len(args) > 1: - code.putln("}") + # actually call the function + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyObjectFastCall", "ObjectHandling.c")) - for i, arg in enumerate(args): - arg.make_owned_reference(code) - code.put_giveref(arg.py_result()) - code.putln("PyTuple_SET_ITEM(%s, %d+%s, %s);" % ( - args_tuple, i, arg_offset, arg.py_result())) - if len(args) > 1: - code.funcstate.release_temp(arg_offset_cname) - - for arg in args: - arg.generate_post_assignment_code(code) - arg.free_temps(code) - - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - function, args_tuple, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) - - code.put_decref_clear(args_tuple, py_object_type) - code.funcstate.release_temp(args_tuple) - - if len(args) == 1: - code.putln("}") - code.putln("}") # !CYTHON_FAST_PYCALL + code.putln("{") + code.putln("PyObject *__pyx_callargs[%d] = {%s, %s};" % ( + len(args)+1, + self_arg, + ', '.join(arg.py_result() for arg in args))) + code.putln("%s = __Pyx_PyObject_FastCall(%s, __pyx_callargs+1-%s, %d+%s);" % ( + self.result(), + function, + arg_offset_cname, + len(args), + arg_offset_cname)) + + code.put_xdecref_clear(self_arg, py_object_type) + code.funcstate.release_temp(self_arg) + code.funcstate.release_temp(arg_offset_cname) + for arg in args: + arg.generate_disposal_code(code) + arg.free_temps(code) + code.putln(code.error_goto_if_null(self.result(), self.pos)) + self.generate_gotref(code) if reuse_function_temp: self.function.generate_disposal_code(code) @@ -5655,6 +6356,7 @@ else: code.put_decref_clear(function, py_object_type) code.funcstate.release_temp(function) + code.putln("}") class InlinedDefNodeCallNode(CallNode): @@ -5705,7 +6407,7 @@ # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) self.args[i] = arg @@ -5752,7 +6454,7 @@ self.function.def_node.entry.pyfunc_cname, arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PythonCapiFunctionNode(ExprNode): @@ -5794,6 +6496,37 @@ SimpleCallNode.__init__(self, pos, **kwargs) +class CachedBuiltinMethodCallNode(CallNode): + # Python call to a method of a known Python builtin (only created in transforms) + + subexprs = ['obj', 'args'] + is_temp = True + + def __init__(self, call_node, obj, method_name, args): + super(CachedBuiltinMethodCallNode, self).__init__( + call_node.pos, + obj=obj, method_name=method_name, args=args, + may_return_none=call_node.may_return_none, + type=call_node.type) + + def may_be_none(self): + if self.may_return_none is not None: + return self.may_return_none + return ExprNode.may_be_none(self) + + def generate_result_code(self, code): + type_cname = self.obj.type.cname + obj_cname = self.obj.py_result() + args = [arg.py_result() for arg in self.args] + call_code = code.globalstate.cached_unbound_method_call_code( + obj_cname, type_cname, self.method_name, args) + code.putln("%s = %s; %s" % ( + self.result(), call_code, + code.error_goto_if_null(self.result(), self.pos) + )) + self.generate_gotref(code) + + class GeneralCallNode(CallNode): # General Python function call, including keyword, # * and ** arguments. @@ -5852,15 +6585,7 @@ self.positional_args = self.positional_args.analyse_types(env) self.positional_args = \ self.positional_args.coerce_to_pyobject(env) - function = self.function - if function.is_name and function.type_entry: - # We are calling an extension type constructor. As long - # as we do not support __new__(), the result type is clear - self.type = function.type_entry.type - self.result_ctype = py_object_type - self.may_return_none = False - else: - self.type = py_object_type + self.set_py_result_type(self.function) self.is_temp = 1 return self @@ -5890,7 +6615,7 @@ kwargs = self.keyword_args declared_args = function_type.args if entry.is_cmethod: - declared_args = declared_args[1:] # skip 'self' + declared_args = declared_args[1:] # skip 'self' if len(pos_args) > len(declared_args): error(self.pos, "function call got too many positional arguments, " @@ -5898,8 +6623,10 @@ len(pos_args))) return None - matched_args = set([ arg.name for arg in declared_args[:len(pos_args)] - if arg.name ]) + matched_args = { + arg.name for arg in declared_args[:len(pos_args)] + if arg.name + } unmatched_args = declared_args[len(pos_args):] matched_kwargs_count = 0 args = list(pos_args) @@ -6017,7 +6744,7 @@ self.positional_args.py_result(), kwargs, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class AsTupleNode(ExprNode): @@ -6027,6 +6754,7 @@ # arg ExprNode subexprs = ['arg'] + is_temp = 1 def calculate_constant_result(self): self.constant_result = tuple(self.arg.constant_result) @@ -6043,7 +6771,6 @@ if self.arg.type is tuple_type: return self.arg.as_none_safe_node("'NoneType' object is not iterable") self.type = tuple_type - self.is_temp = 1 return self def may_be_none(self): @@ -6053,12 +6780,13 @@ gil_message = "Constructing Python tuple" def generate_result_code(self, code): + cfunc = "__Pyx_PySequence_Tuple" if self.arg.type in (py_object_type, tuple_type) else "PySequence_Tuple" code.putln( - "%s = PySequence_Tuple(%s); %s" % ( + "%s = %s(%s); %s" % ( self.result(), - self.arg.py_result(), + cfunc, self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class MergedDictNode(ExprNode): @@ -6116,22 +6844,13 @@ return dict_type def analyse_types(self, env): - args = [ + self.keyword_args = [ arg.analyse_types(env).coerce_to_pyobject(env).as_none_safe_node( # FIXME: CPython's error message starts with the runtime function name 'argument after ** must be a mapping, not NoneType') for arg in self.keyword_args ] - if len(args) == 1 and args[0].type is dict_type: - # strip this intermediate node and use the bare dict - arg = args[0] - if arg.is_name and arg.entry.is_arg and len(arg.entry.cf_assignments) == 1: - # passing **kwargs through to function call => allow NULL - arg.allow_null = True - return arg - - self.keyword_args = args return self def may_be_none(self): @@ -6160,16 +6879,18 @@ self.result(), item.py_result(), code.error_goto_if_null(self.result(), item.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) item.generate_disposal_code(code) if item.type is not dict_type: code.putln('} else {') - code.putln("%s = PyObject_CallFunctionObjArgs((PyObject*)&PyDict_Type, %s, NULL); %s" % ( + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln("%s = __Pyx_PyObject_CallOneArg((PyObject*)&PyDict_Type, %s); %s" % ( self.result(), item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) code.putln('}') item.free_temps(code) @@ -6241,7 +6962,6 @@ is_attribute = 1 subexprs = ['obj'] - type = PyrexTypes.error_type entry = None is_called = 0 needs_none_check = True @@ -6298,7 +7018,11 @@ # FIXME: this is way too redundant with analyse_types() node = self.analyse_as_cimported_attribute_node(env, target=False) if node is not None: - return node.entry.type + if node.entry.type and node.entry.type.is_cfunction: + # special-case - function converted to pointer + return PyrexTypes.CPtrType(node.entry.type) + else: + return node.entry.type node = self.analyse_as_type_attribute(env) if node is not None: return node.entry.type @@ -6309,10 +7033,14 @@ # builtin types cannot be inferred as C functions as # that would prevent their use as bound methods return py_object_type + elif self.entry and self.entry.is_cmethod: + # special case: bound methods should not be inferred + # as their unbound method types + return py_object_type return self.type def analyse_target_declaration(self, env): - pass + self.is_target = True def analyse_target_types(self, env): node = self.analyse_types(env, target = 1) @@ -6323,6 +7051,8 @@ return node def analyse_types(self, env, target = 0): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully self.initialized_check = env.directives['initializedcheck'] node = self.analyse_as_cimported_attribute_node(env, target) if node is None and not target: @@ -6330,7 +7060,7 @@ if node is None: node = self.analyse_as_ordinary_attribute_node(env, target) assert node is not None - if node.entry: + if (node.is_attribute or node.is_name) and node.entry: node.entry.used = True if node.is_attribute: node.wrap_obj_in_nonecheck(env) @@ -6348,6 +7078,10 @@ entry.is_cglobal or entry.is_cfunction or entry.is_type or entry.is_const): return self.as_name_node(env, entry, target) + if self.is_cimported_module_without_shadow(env): + # TODO: search for submodule + error(self.pos, "cimported module has no attribute '%s'" % self.attribute) + return self return None def analyse_as_type_attribute(self, env): @@ -6368,42 +7102,56 @@ return None ubcm_entry = entry else: - # Create a temporary entry describing the C method - # as an ordinary function. - if entry.func_cname and not hasattr(entry.type, 'op_arg_struct'): - cname = entry.func_cname - if entry.type.is_static_method or env.parent_scope.is_cpp_class_scope: - ctype = entry.type - elif type.is_cpp_class: - error(self.pos, "%s not a static member of %s" % (entry.name, type)) - ctype = PyrexTypes.error_type - else: - # Fix self type. - ctype = copy.copy(entry.type) - ctype.args = ctype.args[:] - ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) - else: - cname = "%s->%s" % (type.vtabptr_cname, entry.cname) - ctype = entry.type - ubcm_entry = Symtab.Entry(entry.name, cname, ctype) - ubcm_entry.is_cfunction = 1 - ubcm_entry.func_cname = entry.func_cname - ubcm_entry.is_unbound_cmethod = 1 + ubcm_entry = self._create_unbound_cmethod_entry(type, entry, env) + ubcm_entry.overloaded_alternatives = [ + self._create_unbound_cmethod_entry(type, overloaded_alternative, env) + for overloaded_alternative in entry.overloaded_alternatives + ] return self.as_name_node(env, ubcm_entry, target=False) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: if self.attribute in type.values: - return self.as_name_node(env, env.lookup(self.attribute), target=False) + for entry in type.entry.enum_values: + if entry.name == self.attribute: + return self.as_name_node(env, entry, target=False) + else: + error(self.pos, "%s not a known value of %s" % (self.attribute, type)) else: error(self.pos, "%s not a known value of %s" % (self.attribute, type)) return None + def _create_unbound_cmethod_entry(self, type, entry, env): + # Create a temporary entry describing the unbound C method in `entry` + # as an ordinary function. + if entry.func_cname and entry.type.op_arg_struct is None: + cname = entry.func_cname + if entry.type.is_static_method or ( + env.parent_scope and env.parent_scope.is_cpp_class_scope): + ctype = entry.type + elif type.is_cpp_class: + error(self.pos, "%s not a static member of %s" % (entry.name, type)) + ctype = PyrexTypes.error_type + else: + # Fix self type. + ctype = copy.copy(entry.type) + ctype.args = ctype.args[:] + ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) + else: + cname = "%s->%s" % (type.vtabptr_cname, entry.cname) + ctype = entry.type + ubcm_entry = Symtab.Entry(entry.name, cname, ctype) + ubcm_entry.is_cfunction = 1 + ubcm_entry.func_cname = entry.func_cname + ubcm_entry.is_unbound_cmethod = 1 + ubcm_entry.scope = entry.scope + return ubcm_entry + def analyse_as_type(self, env): module_scope = self.obj.analyse_as_module(env) if module_scope: return module_scope.lookup_type(self.attribute) if not self.obj.is_string_literal: base_type = self.obj.analyse_as_type(env) - if base_type and hasattr(base_type, 'scope') and base_type.scope is not None: + if base_type and getattr(base_type, 'scope', None) is not None: return base_type.scope.lookup_type(self.attribute) return None @@ -6454,13 +7202,18 @@ self.result_ctype = py_object_type elif target and self.obj.type.is_builtin_type: error(self.pos, "Assignment to an immutable object field") + elif self.entry and self.entry.is_cproperty: + if not target: + return SimpleCallNode.for_cproperty(self.pos, self.obj, self.entry).analyse_types(env) + # TODO: implement writable C-properties? + error(self.pos, "Assignment to a read-only property") #elif self.type.is_memoryviewslice and not target: # self.is_temp = True return self def analyse_attribute(self, env, obj_type = None): # Look up attribute and set self.type and self.member. - immutable_obj = obj_type is not None # used during type inference + immutable_obj = obj_type is not None # used during type inference self.is_py_attr = 0 self.member = self.attribute if obj_type is None: @@ -6510,7 +7263,10 @@ # fused function go through assignment synthesis # (foo = pycfunction(foo_func_obj)) and need to go through # regular Python lookup as well - if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: + if entry.is_cproperty: + self.type = entry.type + return + elif (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: self.type = entry.type self.member = entry.cname return @@ -6533,13 +7289,19 @@ self.member = self.attribute self.type = py_object_type self.is_py_attr = 1 + if not obj_type.is_pyobject and not obj_type.is_error: - if obj_type.can_coerce_to_pyobject(env): + # Expose python methods for immutable objects. + if (obj_type.is_string or obj_type.is_cpp_string + or obj_type.is_buffer or obj_type.is_memoryviewslice + or obj_type.is_numeric + or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) + or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) elif (obj_type.is_cfunction and (self.obj.is_name or self.obj.is_attribute) - and self.obj.entry.as_variable - and self.obj.entry.as_variable.type.is_pyobject): + and self.obj.entry.as_variable + and self.obj.entry.as_variable.type.is_pyobject): # might be an optimised builtin function => unpack it if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) @@ -6556,7 +7318,7 @@ format_args = () if (self.obj.type.is_extension_type and self.needs_none_check and not self.is_py_attr): - msg = "'NoneType' object has no attribute '%s'" + msg = "'NoneType' object has no attribute '%{0}s'".format('.30' if len(self.attribute) <= 30 else '') format_args = (self.attribute,) elif self.obj.type.is_memoryviewslice: if self.is_memslice_transpose: @@ -6578,6 +7340,9 @@ gil_message = "Accessing Python attribute" + def is_cimported_module_without_shadow(self, env): + return self.obj.is_cimported_module_without_shadow(env) + def is_simple(self): if self.obj: return self.result_in_temp() or self.obj.is_simple() @@ -6597,9 +7362,14 @@ return NameNode.is_ephemeral(self) def calculate_result_code(self): - #print "AttributeNode.calculate_result_code:", self.member ### - #print "...obj node =", self.obj, "code", self.obj.result() ### - #print "...obj type", self.obj.type, "ctype", self.obj.ctype() ### + result = self.calculate_access_code() + if self.entry and self.entry.is_cpp_optional and not self.is_target: + result = "(*%s)" % result + return result + + def calculate_access_code(self): + # Does the job of calculate_result_code but doesn't dereference cpp_optionals + # Therefore allowing access to the holder variable obj = self.obj obj_code = obj.result_as(obj.type) #print "...obj_code =", obj_code ### @@ -6650,7 +7420,7 @@ self.obj.py_result(), code.intern_identifier(self.attribute), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.type.is_memoryviewslice: if self.is_memslice_transpose: # transpose the slice @@ -6661,10 +7431,11 @@ return code.putln("%s = %s;" % (self.result(), self.obj.result())) - code.put_incref_memoryviewslice(self.result(), have_gil=True) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=True) - T = "__pyx_memslice_transpose(&%s) == 0" - code.putln(code.error_goto_if(T % self.result(), self.pos)) + T = "__pyx_memslice_transpose(&%s)" % self.result() + code.putln(code.error_goto_if_neg(T, self.pos)) elif self.initialized_check: code.putln( 'if (unlikely(!%s.memview)) {' @@ -6672,6 +7443,14 @@ '"Memoryview is not initialized");' '%s' '}' % (self.result(), code.error_goto(self.pos))) + elif self.entry.is_cpp_optional and self.initialized_check: + if self.is_target: + undereferenced_result = self.result() + else: + assert not self.is_temp # calculate_access_code() only makes sense for non-temps + undereferenced_result = self.calculate_access_code() + unbound_check_code = self.type.cpp_optional_check_for_null_code(undereferenced_result) + code.put_error_if_unbound(self.pos, self.entry, unbound_check_code=unbound_check_code) else: # result_code contains what is needed, but we may need to insert # a check and raise an exception @@ -6684,15 +7463,12 @@ def generate_disposal_code(self, code): if self.is_temp and self.type.is_memoryviewslice and self.is_memslice_transpose: # mirror condition for putting the memview incref here: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=True) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_xdecref_clear(self.result(), self.type, have_gil=True) else: ExprNode.generate_disposal_code(self, code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.obj.generate_evaluation_code(code) if self.is_py_attr: code.globalstate.use_utility_code( @@ -6709,12 +7485,14 @@ self.member.upper(), self.obj.result_as(self.obj.type), rhs.result_as(self.ctype()))) + rhs.generate_disposal_code(code) + rhs.free_temps(code) else: select_code = self.result() if self.type.is_pyobject and self.use_managed_ref: rhs.make_owned_reference(code) - code.put_giveref(rhs.py_result()) - code.put_gotref(select_code) + rhs.generate_giveref(code) + code.put_gotref(select_code, self.type) code.put_decref(select_code, self.ctype()) elif self.type.is_memoryviewslice: from . import MemoryView @@ -6725,7 +7503,7 @@ code.putln( "%s = %s;" % ( select_code, - rhs.result_as(self.ctype()))) + rhs.move_result_rhs_as(self.ctype()))) #rhs.result())) rhs.generate_post_assignment_code(code) rhs.free_temps(code) @@ -6754,6 +7532,12 @@ style, text = 'c_attr', 'c attribute (%s)' code.annotate(self.pos, AnnotationItem(style, text % self.type, size=len(self.attribute))) + def get_known_standard_library_import(self): + module_name = self.obj.get_known_standard_library_import() + if module_name: + return StringEncoding.EncodedString("%s.%s" % (module_name, self.attribute)) + return None + #------------------------------------------------------------------- # @@ -6856,9 +7640,10 @@ arg = arg.analyse_types(env) self.args[i] = arg.coerce_to_pyobject(env) if self.mult_factor: - self.mult_factor = self.mult_factor.analyse_types(env) - if not self.mult_factor.type.is_int: - self.mult_factor = self.mult_factor.coerce_to_pyobject(env) + mult_factor = self.mult_factor.analyse_types(env) + if not mult_factor.type.is_int: + mult_factor = mult_factor.coerce_to_pyobject(env) + self.mult_factor = mult_factor.coerce_to_simple(env) self.is_temp = 1 # not setting self.type here, subtypes do this return self @@ -6960,7 +7745,7 @@ len(self.args), ', '.join(arg.py_result() for arg in self.args), code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) elif self.type.is_ctuple: for i, arg in enumerate(self.args): code.putln("%s.f%s = %s;" % ( @@ -6977,7 +7762,7 @@ code.putln("%s = %s(%s%s); %s" % ( target, create_func, arg_count, size_factor, code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) if c_mult: # FIXME: can't use a temp variable here as the code may @@ -7001,7 +7786,7 @@ arg = self.args[i] if c_mult or not arg.result_in_temp(): code.put_incref(arg.result(), arg.ctype()) - code.put_giveref(arg.py_result()) + arg.generate_giveref(code) code.putln("%s(%s, %s, %s);" % ( set_item_func, target, @@ -7018,7 +7803,7 @@ Naming.quick_temp_cname, target, mult_factor.py_result(), code.error_goto_if_null(Naming.quick_temp_cname, self.pos) )) - code.put_gotref(Naming.quick_temp_cname) + code.put_gotref(Naming.quick_temp_cname, py_object_type) code.put_decref(target, py_object_type) code.putln('%s = %s;' % (target, Naming.quick_temp_cname)) code.putln('}') @@ -7040,7 +7825,7 @@ self.mult_factor.generate_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): if self.starred_assignment: self.generate_starred_assignment_code(rhs, code) else: @@ -7102,17 +7887,16 @@ code.putln("PyObject* sequence = %s;" % rhs.py_result()) # list/tuple => check size - code.putln("#if !CYTHON_COMPILING_IN_PYPY") - code.putln("Py_ssize_t size = Py_SIZE(sequence);") - code.putln("#else") - code.putln("Py_ssize_t size = PySequence_Size(sequence);") # < 0 => exception - code.putln("#endif") + code.putln("Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);") code.putln("if (unlikely(size != %d)) {" % len(self.args)) - code.globalstate.use_utility_code(raise_too_many_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c")) code.putln("if (size > %d) __Pyx_RaiseTooManyValuesError(%d);" % ( len(self.args), len(self.args))) - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) code.putln("else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);") + # < 0 => exception code.putln(code.error_goto(self.pos)) code.putln("}") @@ -7139,7 +7923,7 @@ code.putln("%s = PySequence_ITEM(sequence, %d); %s" % ( item.result(), i, code.error_goto_if_null(item.result(), self.pos))) - code.put_gotref(item.result()) + code.put_gotref(item.result(), item.type) else: code.putln("{") code.putln("Py_ssize_t i;") @@ -7149,7 +7933,7 @@ code.putln("for (i=0; i < %s; i++) {" % len(self.unpacked_items)) code.putln("PyObject* item = PySequence_ITEM(sequence, i); %s" % ( code.error_goto_if_null('item', self.pos))) - code.put_gotref('item') + code.put_gotref('item', py_object_type) code.putln("*(temps[i]) = item;") code.putln("}") code.putln("}") @@ -7173,9 +7957,11 @@ code.putln("}") def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True): - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) - code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) - code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) + code.globalstate.use_utility_code( + UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) + code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! if use_loop: code.putln("PyObject** temps[%s] = {%s};" % ( @@ -7188,11 +7974,11 @@ iterator_temp, rhs.py_result(), code.error_goto_if_null(iterator_temp, self.pos))) - code.put_gotref(iterator_temp) + code.put_gotref(iterator_temp, py_object_type) rhs.generate_disposal_code(code) iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False) - code.putln("%s = Py_TYPE(%s)->tp_iternext;" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s);" % ( iternext_func, iterator_temp)) unpacking_error_label = code.new_label('unpacking_failed') @@ -7201,7 +7987,7 @@ code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items)) code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code) code.put_goto(unpacking_error_label) - code.put_gotref("item") + code.put_gotref("item", py_object_type) code.putln("*(temps[index]) = item;") code.putln("}") else: @@ -7213,7 +7999,7 @@ unpack_code, item.result())) code.put_goto(unpacking_error_label) - code.put_gotref(item.py_result()) + item.generate_gotref(code) if terminate: code.globalstate.use_utility_code( @@ -7266,11 +8052,14 @@ starred_target.allocate(code) target_list = starred_target.result() - code.putln("%s = PySequence_List(%s); %s" % ( + code.putln("%s = %s(%s); %s" % ( target_list, + "__Pyx_PySequence_ListKeepNew" if ( + not iterator_temp and rhs.is_temp and rhs.type in (py_object_type, list_type)) + else "PySequence_List", iterator_temp or rhs.py_result(), code.error_goto_if_null(target_list, self.pos))) - code.put_gotref(target_list) + starred_target.generate_gotref(code) if iterator_temp: code.put_decref_clear(iterator_temp, py_object_type) @@ -7279,7 +8068,8 @@ rhs.generate_disposal_code(code) if unpacked_fixed_items_right: - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) length_temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) code.putln('%s = PyList_GET_SIZE(%s);' % (length_temp, target_list)) code.putln("if (unlikely(%s < %d)) {" % (length_temp, len(unpacked_fixed_items_right))) @@ -7301,7 +8091,7 @@ code.putln("%s = PySequence_ITEM(%s, %s-%d); " % ( item.py_result(), target_list, length_temp, i+1)) code.putln('#endif') - code.put_gotref(item.py_result()) + item.generate_gotref(code) coerced_arg.generate_evaluation_code(code) code.putln('#if !CYTHON_COMPILING_IN_CPYTHON') @@ -7309,12 +8099,12 @@ code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % ( sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right), code.error_goto_if_null(sublist_temp, self.pos))) - code.put_gotref(sublist_temp) + code.put_gotref(sublist_temp, py_object_type) code.funcstate.release_temp(length_temp) code.put_decref(target_list, py_object_type) code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp)) code.putln('#else') - code.putln('%s = %s;' % (sublist_temp, sublist_temp)) # avoid warning about unused variable + code.putln('(void)%s;' % sublist_temp) # avoid warning about unused variable code.funcstate.release_temp(sublist_temp) code.putln('#endif') @@ -7343,10 +8133,10 @@ if self.mult_factor or not self.args: return tuple_type arg_types = [arg.infer_type(env) for arg in self.args] - if any(type.is_pyobject or type.is_unspecified or type.is_fused for type in arg_types): + if any(type.is_pyobject or type.is_memoryviewslice or type.is_unspecified or type.is_fused + for type in arg_types): return tuple_type - else: - return env.declare_tuple_type(self.pos, arg_types).type + return env.declare_tuple_type(self.pos, arg_types).type def analyse_types(self, env, skip_children=False): if len(self.args) == 0: @@ -7360,7 +8150,8 @@ arg.starred_expr_allowed_here = True self.args[i] = arg.analyse_types(env) if (not self.mult_factor and - not any((arg.is_starred or arg.type.is_pyobject or arg.type.is_fused) for arg in self.args)): + not any((arg.is_starred or arg.type.is_pyobject or arg.type.is_memoryviewslice or arg.type.is_fused) + for arg in self.args)): self.type = env.declare_tuple_type(self.pos, (arg.type for arg in self.args)).type self.is_temp = 1 return self @@ -7443,26 +8234,26 @@ if len(self.args) == 0: # result_code is Naming.empty_tuple return - if self.is_partly_literal: - # underlying tuple is const, but factor is not - tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2) - const_code = code.get_cached_constants_writer() - const_code.mark_pos(self.pos) - self.generate_sequence_packing_code(const_code, tuple_target, plain=True) - const_code.put_giveref(tuple_target) - code.putln('%s = PyNumber_Multiply(%s, %s); %s' % ( - self.result(), tuple_target, self.mult_factor.py_result(), - code.error_goto_if_null(self.result(), self.pos) + + if self.is_literal or self.is_partly_literal: + # The "mult_factor" is part of the deduplication if it is also constant, i.e. when + # we deduplicate the multiplied result. Otherwise, only deduplicate the constant part. + dedup_key = make_dedup_key(self.type, [self.mult_factor if self.is_literal else None] + self.args) + tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2, dedup_key=dedup_key) + const_code = code.get_cached_constants_writer(tuple_target) + if const_code is not None: + # constant is not yet initialised + const_code.mark_pos(self.pos) + self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal) + const_code.put_giveref(tuple_target, py_object_type) + if self.is_literal: + self.result_code = tuple_target + else: + code.putln('%s = PyNumber_Multiply(%s, %s); %s' % ( + self.result(), tuple_target, self.mult_factor.py_result(), + code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.py_result()) - elif self.is_literal: - # non-empty cached tuple => result is global constant, - # creation code goes into separate code writer - self.result_code = code.get_py_const(py_object_type, 'tuple', cleanup_level=2) - code = code.get_cached_constants_writer() - code.mark_pos(self.pos) - self.generate_sequence_packing_code(code) - code.put_giveref(self.py_result()) + self.generate_gotref(code) else: self.type.entry.used = True self.generate_sequence_packing_code(code) @@ -7484,7 +8275,7 @@ return () def infer_type(self, env): - # TOOD: Infer non-object list arrays. + # TODO: Infer non-object list arrays. return list_type def analyse_expressions(self, env): @@ -7495,11 +8286,10 @@ return node.coerce_to_pyobject(env) def analyse_types(self, env): - hold_errors() - self.original_args = list(self.args) - node = SequenceNode.analyse_types(self, env) - node.obj_conversion_errors = held_errors() - release_errors(ignore=True) + with local_errors(ignore=True) as errors: + self.original_args = list(self.args) + node = SequenceNode.analyse_types(self, env) + node.obj_conversion_errors = errors if env.is_module_scope: self.in_module_scope = True node = node._create_merge_node_if_necessary(env) @@ -7562,20 +8352,18 @@ return t def allocate_temp_result(self, code): - if self.type.is_array and self.in_module_scope: - self.temp_code = code.funcstate.allocate_temp( - self.type, manage_ref=False, static=True) - else: - SequenceNode.allocate_temp_result(self, code) - - def release_temp_result(self, env): if self.type.is_array: - # To be valid C++, we must allocate the memory on the stack - # manually and be sure not to reuse it for something else. - # Yes, this means that we leak a temp array variable. - pass + if self.in_module_scope: + self.temp_code = code.funcstate.allocate_temp( + self.type, manage_ref=False, static=True, reusable=False) + else: + # To be valid C++, we must allocate the memory on the stack + # manually and be sure not to reuse it for something else. + # Yes, this means that we leak a temp array variable. + self.temp_code = code.funcstate.allocate_temp( + self.type, manage_ref=False, reusable=False) else: - SequenceNode.release_temp_result(self, env) + SequenceNode.allocate_temp_result(self, code) def calculate_constant_result(self): if self.mult_factor: @@ -7648,7 +8436,7 @@ if expr_scope is not None: self.expr_scope = expr_scope elif self.has_local_scope: - self.expr_scope = Symtab.GeneratorExpressionScope(outer_scope) + self.expr_scope = Symtab.ComprehensionScope(outer_scope) else: self.expr_scope = None @@ -7677,9 +8465,8 @@ code.putln('{ /* enter inner scope */') py_entries = [] - for entry in self.expr_scope.var_entries: + for _, entry in sorted(item for item in self.expr_scope.entries.items() if item[0]): if not entry.in_closure: - code.put_var_declaration(entry) if entry.type.is_pyobject and entry.used: py_entries.append(entry) if not py_entries: @@ -7689,14 +8476,13 @@ return # must free all local Python references at each exit point - old_loop_labels = tuple(code.new_loop_labels()) + old_loop_labels = code.new_loop_labels() old_error_label = code.new_error_label() generate_inner_evaluation_code(code) # normal (non-error) exit - for entry in py_entries: - code.put_var_decref(entry) + self._generate_vars_cleanup(code, py_entries) # error/loop body exit points exit_scope = code.new_label('exit_scope') @@ -7705,8 +8491,7 @@ list(zip(code.get_loop_labels(), old_loop_labels))): if code.label_used(label): code.put_label(label) - for entry in py_entries: - code.put_var_decref(entry) + self._generate_vars_cleanup(code, py_entries) code.put_goto(old_label) code.put_label(exit_scope) code.putln('} /* exit inner scope */') @@ -7714,6 +8499,14 @@ code.set_loop_labels(old_loop_labels) code.error_label = old_error_label + def _generate_vars_cleanup(self, code, py_entries): + for entry in py_entries: + if entry.is_cglobal: + code.put_var_gotref(entry) + code.put_var_decref_set(entry, "Py_None") + else: + code.put_var_xdecref_clear(entry) + class ComprehensionNode(ScopedExprNode): # A list/set/dict comprehension @@ -7721,12 +8514,13 @@ child_attrs = ["loop"] is_temp = True + constant_result = not_a_constant def infer_type(self, env): return self.type def analyse_declarations(self, env): - self.append.target = self # this is used in the PyList_Append of the inner loop + self.append.target = self # this is used in the PyList_Append of the inner loop self.init_scope(env) def analyse_scoped_declarations(self, env): @@ -7761,7 +8555,7 @@ self.result(), create_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) self.loop.generate_execution_code(code) def annotate(self, code): @@ -7886,7 +8680,7 @@ code.putln("%s = __Pyx_Generator_Next(%s); %s" % ( self.result(), self.gen.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class MergedSequenceNode(ExprNode): @@ -7994,10 +8788,12 @@ else: code.putln("%s = %s(%s); %s" % ( self.result(), - 'PySet_New' if is_set else 'PySequence_List', + 'PySet_New' if is_set + else "__Pyx_PySequence_ListKeepNew" if item.is_temp and item.type in (py_object_type, list_type) + else "PySequence_List", item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) item.free_temps(code) @@ -8047,7 +8843,7 @@ self.result(), Naming.quick_temp_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) code.putln("}") for helper in sorted(helpers): @@ -8080,7 +8876,7 @@ return False def calculate_constant_result(self): - self.constant_result = set([arg.constant_result for arg in self.args]) + self.constant_result = {arg.constant_result for arg in self.args} def compile_time_value(self, denv): values = [arg.compile_time_value(denv) for arg in self.args] @@ -8097,7 +8893,7 @@ "%s = PySet_New(0); %s" % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for arg in self.args: code.put_error_if_neg( self.pos, @@ -8144,15 +8940,16 @@ return () def infer_type(self, env): - # TOOD: Infer struct constructors. + # TODO: Infer struct constructors. return dict_type def analyse_types(self, env): - hold_errors() - self.key_value_pairs = [ item.analyse_types(env) - for item in self.key_value_pairs ] - self.obj_conversion_errors = held_errors() - release_errors(ignore=True) + with local_errors(ignore=True) as errors: + self.key_value_pairs = [ + item.analyse_types(env) + for item in self.key_value_pairs + ] + self.obj_conversion_errors = errors return self def may_be_none(self): @@ -8183,7 +8980,7 @@ error(item.key.pos, "Invalid struct field identifier") item.key = StringNode(item.key.pos, value="") else: - key = str(item.key.value) # converts string literals to unicode in Py3 + key = str(item.key.value) # converts string literals to unicode in Py3 member = dst_type.scope.lookup_here(key) if not member: error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key)) @@ -8193,8 +8990,7 @@ value = value.arg item.value = value.coerce_to(member.type, env) else: - self.type = error_type - error(self.pos, "Cannot interpret dict as type '%s'" % dst_type) + return super(DictNode, self).coerce_to(dst_type, env) return self def release_errors(self): @@ -8214,10 +9010,11 @@ if is_dict: self.release_errors() code.putln( - "%s = PyDict_New(); %s" % ( + "%s = __Pyx_PyDict_NewPresized(%d); %s" % ( self.result(), + len(self.key_value_pairs), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) keys_seen = set() key_type = None @@ -8281,6 +9078,11 @@ for item in self.key_value_pairs: item.annotate(code) + def as_python_dict(self): + # returns a dict with constant keys and Node values + # (only works on DictNodes where the keys are ConstNodes or PyConstNode) + return dict([(key.value, value) for key, value in self.key_value_pairs]) + class DictItemNode(ExprNode): # Represents a single item in a DictNode @@ -8289,7 +9091,7 @@ # value ExprNode subexprs = ['key', 'value'] - nogil_check = None # Parent DictNode takes care of it + nogil_check = None # Parent DictNode takes care of it def calculate_constant_result(self): self.constant_result = ( @@ -8345,7 +9147,7 @@ code.putln('%s = PyDict_Keys(%s); %s' % ( self.result(), dict_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: # originally used PyMapping_Keys() here, but that may return a tuple code.globalstate.use_utility_code(UtilityCode.load_cached( @@ -8354,11 +9156,11 @@ code.putln('%s = __Pyx_PyObject_CallMethod0(%s, %s); %s' % ( self.result(), dict_result, keys_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("if (unlikely(!PyList_Check(%s))) {" % self.result()) - code.put_decref_set(self.result(), "PySequence_List(%s)" % self.result()) + self.generate_decref_set(code, "PySequence_List(%s)" % self.result()) code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("}") code.put_error_if_neg( self.pos, 'PyList_Sort(%s)' % self.py_result()) @@ -8380,21 +9182,22 @@ # a name, tuple of bases and class dictionary. # # name EncodedString Name of the class - # bases ExprNode Base class tuple - # dict ExprNode Class dict (not owned by this node) + # class_def_node PyClassDefNode PyClassDefNode defining this class # doc ExprNode or None Doc string # module_name EncodedString Name of defining module - subexprs = ['bases', 'doc'] + subexprs = ['doc'] type = py_object_type is_temp = True + def analyse_annotations(self, env): + pass + def infer_type(self, env): # TODO: could return 'type' in some cases return py_object_type def analyse_types(self, env): - self.bases = self.bases.analyse_types(env) if self.doc: self.doc = self.doc.analyse_types(env) self.doc = self.doc.coerce_to_pyobject(env) @@ -8407,12 +9210,13 @@ gil_message = "Constructing Python class" def generate_result_code(self, code): + class_def_node = self.class_def_node cname = code.intern_identifier(self.name) if self.doc: code.put_error_if_neg(self.pos, 'PyDict_SetItem(%s, %s, %s)' % ( - self.dict.py_result(), + class_def_node.dict.py_result(), code.intern_identifier( StringEncoding.EncodedString("__doc__")), self.doc.py_result())) @@ -8421,13 +9225,13 @@ code.putln( '%s = __Pyx_CreateClass(%s, %s, %s, %s, %s); %s' % ( self.result(), - self.bases.py_result(), - self.dict.py_result(), + class_def_node.bases.py_result(), + class_def_node.dict.py_result(), cname, qualname, py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class Py3ClassNode(ExprNode): @@ -8436,13 +9240,15 @@ # a name, tuple of bases and class dictionary. # # name EncodedString Name of the class - # dict ExprNode Class dict (not owned by this node) # module_name EncodedString Name of defining module + # class_def_node PyClassDefNode PyClassDefNode defining this class # calculate_metaclass bool should call CalculateMetaclass() # allow_py2_metaclass bool should look for Py2 metaclass + # force_type bool always create a "new style" class, even with no bases subexprs = [] type = py_object_type + force_type = False is_temp = True def infer_type(self, env): @@ -8457,15 +9263,35 @@ gil_message = "Constructing Python class" + def analyse_annotations(self, env): + from .AutoDocTransforms import AnnotationWriter + position = self.class_def_node.pos + dict_items = [ + DictItemNode( + entry.pos, + key=IdentifierStringNode(entry.pos, value=entry.name), + value=entry.annotation.string + ) + for entry in env.entries.values() if entry.annotation + ] + # Annotations dict shouldn't exist for classes which don't declare any. + if dict_items: + annotations_dict = DictNode(position, key_value_pairs=dict_items) + lhs = NameNode(position, name=StringEncoding.EncodedString(u"__annotations__")) + lhs.entry = env.lookup_here(lhs.name) or env.declare_var(lhs.name, dict_type, position) + node = SingleAssignmentNode(position, lhs=lhs, rhs=annotations_dict) + node.analyse_declarations(env) + self.class_def_node.body.stats.insert(0, node) + def generate_result_code(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c")) cname = code.intern_identifier(self.name) - if self.mkw: - mkw = self.mkw.py_result() - else: - mkw = 'NULL' - if self.metaclass: - metaclass = self.metaclass.result() + class_def_node = self.class_def_node + mkw = class_def_node.mkw.py_result() if class_def_node.mkw else 'NULL' + if class_def_node.metaclass: + metaclass = class_def_node.metaclass.py_result() + elif self.force_type: + metaclass = "((PyObject*)&PyType_Type)" else: metaclass = "((PyObject*)&__Pyx_DefaultClassType)" code.putln( @@ -8473,20 +9299,19 @@ self.result(), metaclass, cname, - self.bases.py_result(), - self.dict.py_result(), + class_def_node.bases.py_result(), + class_def_node.dict.py_result(), mkw, self.calculate_metaclass, self.allow_py2_metaclass, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassMetaclassNode(ExprNode): # Helper class holds Python3 metaclass object # - # bases ExprNode Base class tuple (not owned by this node) - # mkw ExprNode Class keyword arguments (not owned by this node) + # class_def_node PyClassDefNode PyClassDefNode defining this class subexprs = [] @@ -8499,38 +9324,38 @@ return True def generate_result_code(self, code): - if self.mkw: + bases = self.class_def_node.bases + mkw = self.class_def_node.mkw + if mkw: code.globalstate.use_utility_code( UtilityCode.load_cached("Py3MetaclassGet", "ObjectHandling.c")) call = "__Pyx_Py3MetaclassGet(%s, %s)" % ( - self.bases.result(), - self.mkw.result()) + bases.result(), + mkw.result()) else: code.globalstate.use_utility_code( UtilityCode.load_cached("CalculateMetaclass", "ObjectHandling.c")) call = "__Pyx_CalculateMetaclass(NULL, %s)" % ( - self.bases.result()) + bases.result()) code.putln( "%s = %s; %s" % ( self.result(), call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) + class PyClassNamespaceNode(ExprNode, ModuleNameMixin): # Helper class holds Python3 namespace object # # All this are not owned by this node - # metaclass ExprNode Metaclass object - # bases ExprNode Base class tuple - # mkw ExprNode Class keyword arguments + # class_def_node PyClassDefNode PyClassDefNode defining this class # doc ExprNode or None Doc string (owned) subexprs = ['doc'] def analyse_types(self, env): if self.doc: - self.doc = self.doc.analyse_types(env) - self.doc = self.doc.coerce_to_pyobject(env) + self.doc = self.doc.analyse_types(env).coerce_to_pyobject(env) self.type = py_object_type self.is_temp = 1 return self @@ -8542,30 +9367,23 @@ cname = code.intern_identifier(self.name) py_mod_name = self.get_py_mod_name(code) qualname = self.get_py_qualified_name(code) - if self.doc: - doc_code = self.doc.result() - else: - doc_code = '(PyObject *) NULL' - if self.mkw: - mkw = self.mkw.py_result() - else: - mkw = '(PyObject *) NULL' - if self.metaclass: - metaclass = self.metaclass.result() - else: - metaclass = "(PyObject *) NULL" + class_def_node = self.class_def_node + null = "(PyObject *) NULL" + doc_code = self.doc.result() if self.doc else null + mkw = class_def_node.mkw.py_result() if class_def_node.mkw else null + metaclass = class_def_node.metaclass.py_result() if class_def_node.metaclass else null code.putln( "%s = __Pyx_Py3MetaclassPrepare(%s, %s, %s, %s, %s, %s, %s); %s" % ( self.result(), metaclass, - self.bases.result(), + class_def_node.bases.result(), cname, qualname, mkw, py_mod_name, doc_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ClassCellInjectorNode(ExprNode): @@ -8576,110 +9394,48 @@ is_active = False def analyse_expressions(self, env): - if self.is_active: - env.use_utility_code( - UtilityCode.load_cached("CyFunctionClassCell", "CythonFunction.c")) return self - def generate_evaluation_code(self, code): - if self.is_active: - self.allocate_temp_result(code) - code.putln( - '%s = PyList_New(0); %s' % ( - self.result(), - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) - - def generate_injection_code(self, code, classobj_cname): - if self.is_active: - code.put_error_if_neg(self.pos, '__Pyx_CyFunction_InitClassCell(%s, %s)' % ( - self.result(), classobj_cname)) - - -class ClassCellNode(ExprNode): - # Class Cell for noargs super() - subexprs = [] - is_temp = True - is_generator = False - type = py_object_type - - def analyse_types(self, env): - return self - - def generate_result_code(self, code): - if not self.is_generator: - code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % ( - self.result(), - Naming.self_cname)) - else: - code.putln('%s = %s->classobj;' % ( - self.result(), Naming.generator_cname)) - code.putln( - 'if (!%s) { PyErr_SetString(PyExc_SystemError, ' - '"super(): empty __class__ cell"); %s }' % ( - self.result(), - code.error_goto(self.pos))) - code.put_incref(self.result(), py_object_type) - - -class BoundMethodNode(ExprNode): - # Helper class used in the implementation of Python - # class definitions. Constructs an bound method - # object from a class and a function. - # - # function ExprNode Function object - # self_object ExprNode self object - - subexprs = ['function'] - - def analyse_types(self, env): - self.function = self.function.analyse_types(env) - self.type = py_object_type - self.is_temp = 1 - return self - - gil_message = "Constructing a bound method" - def generate_result_code(self, code): + assert self.is_active code.putln( - "%s = __Pyx_PyMethod_New(%s, %s, (PyObject*)%s->ob_type); %s" % ( + '%s = PyList_New(0); %s' % ( self.result(), - self.function.py_result(), - self.self_object.py_result(), - self.self_object.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) + + def generate_injection_code(self, code, classobj_cname): + assert self.is_active + code.globalstate.use_utility_code( + UtilityCode.load_cached("CyFunctionClassCell", "CythonFunction.c")) + code.put_error_if_neg(self.pos, '__Pyx_CyFunction_InitClassCell(%s, %s)' % ( + self.result(), classobj_cname)) -class UnboundMethodNode(ExprNode): - # Helper class used in the implementation of Python - # class definitions. Constructs an unbound method - # object from a class and a function. - # - # function ExprNode Function object +class ClassCellNode(ExprNode): + # Class Cell for noargs super() + subexprs = [] + is_temp = True + is_generator = False type = py_object_type - is_temp = 1 - - subexprs = ['function'] def analyse_types(self, env): - self.function = self.function.analyse_types(env) return self - def may_be_none(self): - return False - - gil_message = "Constructing an unbound method" - def generate_result_code(self, code): - class_cname = code.pyclass_stack[-1].classobj.result() + if not self.is_generator: + code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % ( + self.result(), + Naming.self_cname)) + else: + code.putln('%s = %s->classobj;' % ( + self.result(), Naming.generator_cname)) code.putln( - "%s = __Pyx_PyMethod_New(%s, 0, %s); %s" % ( + 'if (!%s) { PyErr_SetString(PyExc_SystemError, ' + '"super(): empty __class__ cell"); %s }' % ( self.result(), - self.function.py_result(), - class_cname, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + code.error_goto(self.pos))) + code.put_incref(self.result(), py_object_type) class PyCFunctionNode(ExprNode, ModuleNameMixin): @@ -8688,7 +9444,6 @@ # from a PyMethodDef struct. # # pymethdef_cname string PyMethodDef structure - # self_object ExprNode or None # binding bool # def_node DefNode the Python function node # module_name EncodedString Name of defining module @@ -8697,7 +9452,6 @@ subexprs = ['code_object', 'defaults_tuple', 'defaults_kwdict', 'annotations_dict'] - self_object = None code_object = None binding = False def_node = None @@ -8746,36 +9500,35 @@ must_use_constants = env.is_c_class_scope or (self.def_node.is_wrapper and env.is_module_scope) for arg in self.def_node.args: - if arg.default and not must_use_constants: - if not arg.default.is_literal: - arg.is_dynamic = True - if arg.type.is_pyobject: - nonliteral_objects.append(arg) + if arg.default: + if not must_use_constants: + if arg.default.is_literal: + arg.default = DefaultLiteralArgNode(arg.pos, arg.default) else: - nonliteral_other.append(arg) - else: - arg.default = DefaultLiteralArgNode(arg.pos, arg.default) - if arg.kw_only: - default_kwargs.append(arg) - else: - default_args.append(arg) + arg.is_dynamic = True + if arg.type.is_pyobject: + nonliteral_objects.append(arg) + else: + nonliteral_other.append(arg) + if arg.default.type and arg.default.type.can_coerce_to_pyobject(env): + if arg.kw_only: + default_kwargs.append(arg) + else: + default_args.append(arg) if arg.annotation: arg.annotation = arg.annotation.analyse_types(env) - if not arg.annotation.type.is_pyobject: - arg.annotation = arg.annotation.coerce_to_pyobject(env) - annotations.append((arg.pos, arg.name, arg.annotation)) + annotations.append((arg.pos, arg.name, arg.annotation.string)) for arg in (self.def_node.star_arg, self.def_node.starstar_arg): if arg and arg.annotation: arg.annotation = arg.annotation.analyse_types(env) - if not arg.annotation.type.is_pyobject: - arg.annotation = arg.annotation.coerce_to_pyobject(env) - annotations.append((arg.pos, arg.name, arg.annotation)) - - if self.def_node.return_type_annotation: - annotations.append((self.def_node.return_type_annotation.pos, - StringEncoding.EncodedString("return"), - self.def_node.return_type_annotation)) + annotations.append((arg.pos, arg.name, arg.annotation.string)) + + annotation = self.def_node.return_type_annotation + if annotation: + self.def_node.return_type_annotation = annotation.analyse_types(env) + annotations.append((annotation.pos, StringEncoding.EncodedString("return"), + annotation.string)) if nonliteral_objects or nonliteral_other: module_scope = env.global_scope() @@ -8783,14 +9536,17 @@ scope = Symtab.StructOrUnionScope(cname) self.defaults = [] for arg in nonliteral_objects: - entry = scope.declare_var(arg.name, arg.type, None, + type_ = arg.type + if type_.is_buffer: + type_ = type_.base + entry = scope.declare_var(arg.name, type_, None, Naming.arg_prefix + arg.name, allow_pyobject=True) self.defaults.append((arg, entry)) for arg in nonliteral_other: entry = scope.declare_var(arg.name, arg.type, None, Naming.arg_prefix + arg.name, - allow_pyobject=False) + allow_pyobject=False, allow_memoryview=True) self.defaults.append((arg, entry)) entry = module_scope.declare_struct_or_union( None, 'struct', scope, 1, None, cname=cname) @@ -8815,7 +9571,8 @@ value=arg.default) for arg in default_kwargs]) self.defaults_kwdict = defaults_kwdict.analyse_types(env) - else: + elif not self.specialized_cpdefs: + # Fused dispatch functions do not support (dynamic) default arguments, only the specialisations do. if default_args: defaults_tuple = DefaultsTupleNode( self.pos, default_args, self.defaults_struct) @@ -8857,12 +9614,8 @@ gil_message = "Constructing Python function" - def self_result_code(self): - if self.self_object is None: - self_result = "NULL" - else: - self_result = self.self_object.py_result() - return self_result + def closure_result_code(self): + return "NULL" def generate_result_code(self, code): if self.binding: @@ -8876,11 +9629,11 @@ '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % ( self.result(), self.pymethdef_cname, - self.self_result_code(), + self.closure_result_code(), py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_cyfunction_code(self, code): if self.specialized_cpdefs: @@ -8891,11 +9644,11 @@ if self.specialized_cpdefs or self.is_specialization: code.globalstate.use_utility_code( UtilityCode.load_cached("FusedFunction", "CythonFunction.c")) - constructor = "__pyx_FusedFunction_NewEx" + constructor = "__pyx_FusedFunction_New" else: code.globalstate.use_utility_code( UtilityCode.load_cached("CythonFunction", "CythonFunction.c")) - constructor = "__Pyx_CyFunction_NewEx" + constructor = "__Pyx_CyFunction_New" if self.code_object: code_object_result = self.code_object.py_result() @@ -8911,6 +9664,9 @@ if def_node.local_scope.parent_scope.is_c_class_scope and not def_node.entry.is_anonymous: flags.append('__Pyx_CYFUNCTION_CCLASS') + if def_node.is_coroutine: + flags.append('__Pyx_CYFUNCTION_COROUTINE') + if flags: flags = ' | '.join(flags) else: @@ -8923,13 +9679,13 @@ self.pymethdef_cname, flags, self.get_py_qualified_name(code), - self.self_result_code(), + self.closure_result_code(), self.get_py_mod_name(code), Naming.moddict_cname, code_object_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if def_node.requires_classobj: assert code.pyclass_stack, "pyclass_stack is empty" @@ -8939,7 +9695,7 @@ 'PyList_Append(%s, %s);' % ( class_node.class_cell.result(), self.result())) - code.put_giveref(self.py_result()) + self.generate_giveref(code) if self.defaults: code.putln( @@ -8955,26 +9711,29 @@ if self.defaults_tuple: code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % ( self.result(), self.defaults_tuple.py_result())) - if self.defaults_kwdict: - code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( - self.result(), self.defaults_kwdict.py_result())) - if def_node.defaults_getter: - code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( - self.result(), def_node.defaults_getter.entry.pyfunc_cname)) - if self.annotations_dict: - code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( - self.result(), self.annotations_dict.py_result())) + if not self.specialized_cpdefs: + # disable introspection functions for fused dispatcher function since the user never sees it + # TODO: this is mostly disabled because the attributes end up pointing to ones belonging + # to the specializations - ideally this would be fixed instead + if self.defaults_kwdict: + code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( + self.result(), self.defaults_kwdict.py_result())) + if def_node.defaults_getter: + code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( + self.result(), def_node.defaults_getter.entry.pyfunc_cname)) + if self.annotations_dict: + code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( + self.result(), self.annotations_dict.py_result())) class InnerFunctionNode(PyCFunctionNode): # Special PyCFunctionNode that depends on a closure class - # binding = True - needs_self_code = True + needs_closure_code = True - def self_result_code(self): - if self.needs_self_code: + def closure_result_code(self): + if self.needs_closure_code: return "((PyObject*)%s)" % Naming.cur_scope_cname return "NULL" @@ -9013,7 +9772,9 @@ if self.result_code is None: self.result_code = code.get_py_const(py_object_type, 'codeobj', cleanup_level=2) - code = code.get_cached_constants_writer() + code = code.get_cached_constants_writer(self.result_code) + if code is None: + return # already initialised code.mark_pos(self.pos) func = self.def_node func_name = code.get_py_string_const( @@ -9022,15 +9783,18 @@ file_path = StringEncoding.bytes_literal(func.pos[0].get_filenametable_entry().encode('utf8'), 'utf8') file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True) - flags = [] + # This combination makes CPython create a new dict for "frame.f_locals" (see GH #1836). + flags = ['CO_OPTIMIZED', 'CO_NEWLOCALS'] + if self.def_node.star_arg: flags.append('CO_VARARGS') if self.def_node.starstar_arg: flags.append('CO_VARKEYWORDS') - code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( + code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( self.result_code, len(func.args) - func.num_kwonly_args, # argcount + func.num_posonly_args, # posonlyargcount (Py3.8+ only) func.num_kwonly_args, # kwonlyargcount (Py3 only) len(self.varnames.args), # nlocals '|'.join(flags) or '0', # flags @@ -9149,11 +9913,14 @@ name = StringEncoding.EncodedString('') def analyse_declarations(self, env): + if hasattr(self, "lambda_name"): + # this if-statement makes it safe to run twice + return self.lambda_name = self.def_node.lambda_name = env.next_id('lambda') self.def_node.no_assignment_synthesis = True self.def_node.pymethdef_required = True - self.def_node.analyse_declarations(env) self.def_node.is_cyfunction = True + self.def_node.analyse_declarations(env) self.pymethdef_cname = self.def_node.entry.pymethdef_cname env.add_lambda_def(self.def_node) @@ -9178,6 +9945,9 @@ binding = False def analyse_declarations(self, env): + if hasattr(self, "genexpr_name"): + # this if-statement makes it safe to run twice + return self.genexpr_name = env.next_id('genexpr') super(GeneratorExpressionNode, self).analyse_declarations(env) # No pymethdef required @@ -9192,9 +9962,9 @@ '%s = %s(%s); %s' % ( self.result(), self.def_node.entry.pyfunc_cname, - self.self_result_code(), + self.closure_result_code(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class YieldExprNode(ExprNode): @@ -9209,10 +9979,11 @@ label_num = 0 is_yield_from = False is_await = False + in_async_gen = False expr_keyword = 'yield' def analyse_types(self, env): - if not self.label_num: + if not self.label_num or (self.is_yield_from and self.in_async_gen): error(self.pos, "'%s' not supported here" % self.expr_keyword) self.is_temp = 1 if self.arg is not None: @@ -9243,7 +10014,8 @@ Generate the code to return the argument in 'Naming.retval_cname' and to continue at the yield label. """ - label_num, label_name = code.new_yield_label() + label_num, label_name = code.new_yield_label( + self.expr_keyword.replace(' ', '_')) code.use_label(label_name) saved = [] @@ -9251,44 +10023,62 @@ for cname, type, manage_ref in code.funcstate.temps_in_use(): save_cname = code.funcstate.closure_temps.allocate_temp(type) saved.append((cname, save_cname, type)) - if type.is_pyobject: - code.put_xgiveref(cname) + if type.is_cpp_class: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % cname + else: + code.put_xgiveref(cname, type) code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname)) - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) + profile = code.globalstate.directives['profile'] + linetrace = code.globalstate.directives['linetrace'] + if profile or linetrace: + code.put_trace_return(Naming.retval_cname, + nogil=not code.funcstate.gil_owned) code.put_finish_refcount_context() - code.putln("/* return from generator, yielding value */") + + if code.funcstate.current_except is not None: + # inside of an except block => save away currently handled exception + code.putln("__Pyx_Coroutine_SwapException(%s);" % Naming.generator_cname) + else: + # no exceptions being handled => restore exception state of caller + code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname) + + code.putln("/* return from %sgenerator, %sing value */" % ( + 'async ' if self.in_async_gen else '', + 'await' if self.is_await else 'yield')) code.putln("%s->resume_label = %d;" % ( Naming.generator_cname, label_num)) - code.putln("return %s;" % Naming.retval_cname) + if self.in_async_gen and not self.is_await: + # __Pyx__PyAsyncGenValueWrapperNew() steals a reference to the return value + code.putln("return __Pyx__PyAsyncGenValueWrapperNew(%s);" % Naming.retval_cname) + else: + code.putln("return %s;" % Naming.retval_cname) code.put_label(label_name) for cname, save_cname, type in saved: - code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname)) + save_cname = "%s->%s" % (Naming.cur_scope_cname, save_cname) + if type.is_cpp_class: + save_cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % save_cname + code.putln('%s = %s;' % (cname, save_cname)) if type.is_pyobject: - code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname)) - code.put_xgotref(cname) - code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos)) + code.putln('%s = 0;' % save_cname) + code.put_xgotref(cname, type) + self.generate_sent_value_handling_code(code, Naming.sent_value_cname) if self.result_is_used: self.allocate_temp_result(code) code.put('%s = %s; ' % (self.result(), Naming.sent_value_cname)) code.put_incref(self.result(), py_object_type) + def generate_sent_value_handling_code(self, code, value_cname): + code.putln(code.error_goto_if_null(value_cname, self.pos)) -class YieldFromExprNode(YieldExprNode): - # "yield from GEN" expression - is_yield_from = True - expr_keyword = 'yield from' - - def coerce_yield_argument(self, env): - if not self.arg.type.is_string: - # FIXME: support C arrays and C++ iterators? - error(self.pos, "yielding from non-Python object not supported") - self.arg = self.arg.coerce_to_pyobject(env) +class _YieldDelegationExprNode(YieldExprNode): def yield_from_func(self, code): - code.globalstate.use_utility_code(UtilityCode.load_cached("GeneratorYieldFrom", "Coroutine.c")) - return "__Pyx_Generator_Yield_From" + raise NotImplementedError() def generate_evaluation_code(self, code, source_cname=None, decref_source=False): if source_cname is None: @@ -9303,7 +10093,7 @@ self.arg.free_temps(code) elif decref_source: code.put_decref_clear(source_cname, py_object_type) - code.put_xgotref(Naming.retval_cname) + code.put_xgotref(Naming.retval_cname, py_object_type) code.putln("if (likely(%s)) {" % Naming.retval_cname) self.generate_yield_code(code) @@ -9319,18 +10109,34 @@ # YieldExprNode has allocated the result temp for us code.putln("%s = NULL;" % self.result()) code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result()) - code.put_gotref(self.result()) + self.generate_gotref(code) def handle_iteration_exception(self, code): - code.putln("PyObject* exc_type = PyErr_Occurred();") + code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") code.putln("if (exc_type) {") - code.putln("if (likely(exc_type == PyExc_StopIteration ||" - " PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();") + code.putln("if (likely(exc_type == PyExc_StopIteration || (exc_type != PyExc_GeneratorExit &&" + " __Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)))) PyErr_Clear();") code.putln("else %s" % code.error_goto(self.pos)) code.putln("}") -class AwaitExprNode(YieldFromExprNode): +class YieldFromExprNode(_YieldDelegationExprNode): + # "yield from GEN" expression + is_yield_from = True + expr_keyword = 'yield from' + + def coerce_yield_argument(self, env): + if not self.arg.type.is_string: + # FIXME: support C arrays and C++ iterators? + error(self.pos, "yielding from non-Python object not supported") + self.arg = self.arg.coerce_to_pyobject(env) + + def yield_from_func(self, code): + code.globalstate.use_utility_code(UtilityCode.load_cached("GeneratorYieldFrom", "Coroutine.c")) + return "__Pyx_Generator_Yield_From" + + +class AwaitExprNode(_YieldDelegationExprNode): # 'await' expression node # # arg ExprNode the Awaitable value to await @@ -9349,29 +10155,34 @@ return "__Pyx_Coroutine_Yield_From" -class AIterAwaitExprNode(AwaitExprNode): - # 'await' expression node used in async-for loops to support the pre-Py3.5.2 'aiter' protocol - def yield_from_func(self, code): - code.globalstate.use_utility_code(UtilityCode.load_cached("CoroutineAIterYieldFrom", "Coroutine.c")) - return "__Pyx_Coroutine_AIter_Yield_From" - - class AwaitIterNextExprNode(AwaitExprNode): # 'await' expression node as part of 'async for' iteration # # Breaks out of loop on StopAsyncIteration exception. - def fetch_iteration_result(self, code): - assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop" + def _generate_break(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c")) - code.putln("PyObject* exc_type = PyErr_Occurred();") - code.putln("if (exc_type && likely(exc_type == __Pyx_PyExc_StopAsyncIteration ||" - " PyErr_GivenExceptionMatches(exc_type, __Pyx_PyExc_StopAsyncIteration))) {") + code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") + code.putln("if (unlikely(exc_type && (exc_type == __Pyx_PyExc_StopAsyncIteration || (" + " exc_type != PyExc_StopIteration && exc_type != PyExc_GeneratorExit &&" + " __Pyx_PyErr_GivenExceptionMatches(exc_type, __Pyx_PyExc_StopAsyncIteration))))) {") code.putln("PyErr_Clear();") code.putln("break;") code.putln("}") + + def fetch_iteration_result(self, code): + assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop" + self._generate_break(code) super(AwaitIterNextExprNode, self).fetch_iteration_result(code) + def generate_sent_value_handling_code(self, code, value_cname): + assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop" + code.putln("if (unlikely(!%s)) {" % value_cname) + self._generate_break(code) + # all non-break exceptions are errors, as in parent class + code.putln(code.error_goto(self.pos)) + code.putln("}") + class GlobalsExprNode(AtomicExprNode): type = dict_type @@ -9390,7 +10201,7 @@ code.putln('%s = __Pyx_Globals(); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class LocalsDictItemNode(DictItemNode): @@ -9517,7 +10328,10 @@ def analyse_types(self, env): self.operand = self.operand.analyse_types(env) - if self.is_py_operation(): + if self.is_pythran_operation(env): + self.type = PythranExpr(pythran_unaryop_type(self.operator, self.operand.type)) + self.is_temp = 1 + elif self.is_py_operation(): self.coerce_operand_to_pyobject(env) self.type = py_object_type self.is_temp = 1 @@ -9533,6 +10347,11 @@ def is_py_operation(self): return self.operand.type.is_pyobject or self.operand.type.is_ctuple + def is_pythran_operation(self, env): + np_pythran = has_np_pythran(env) + op_type = self.operand.type + return np_pythran and (op_type.is_buffer or op_type.is_pythran_expr) + def nogil_check(self, env): if self.is_py_operation(): self.gil_error() @@ -9545,12 +10364,21 @@ self.operand = self.operand.coerce_to_pyobject(env) def generate_result_code(self, code): - if self.operand.type.is_pyobject: + if self.type.is_pythran_expr: + code.putln("// Pythran unaryop") + code.putln("__Pyx_call_destructor(%s);" % self.result()) + code.putln("new (&%s) decltype(%s){%s%s};" % ( + self.result(), + self.result(), + self.operator, + self.operand.pythran_result())) + elif self.operand.type.is_pyobject: self.generate_py_operation_code(code) elif self.is_temp: if self.is_cpp_operation() and self.exception_check == '+': translate_cpp_exception(code, self.pos, "%s = %s %s;" % (self.result(), self.operator, self.operand.result()), + self.result() if self.type.is_pyobject else None, self.exception_value, self.in_nogil_context) else: code.putln("%s = %s %s;" % (self.result(), self.operator, self.operand.result())) @@ -9563,7 +10391,7 @@ function, self.operand.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def type_error(self): if not self.operand.type.is_error: @@ -9581,7 +10409,7 @@ self.exception_value = entry.type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) else: self.exception_check = '' @@ -9715,7 +10543,10 @@ def analyse_c_operation(self, env): if self.operand.type.is_ptr: - self.type = self.operand.type.base_type + if env.is_cpp: + self.type = PyrexTypes.CReferenceType(self.operand.type.base_type) + else: + self.type = self.operand.type.base_type else: self.type_error() @@ -9766,7 +10597,10 @@ self.error("Taking address of non-lvalue (type %s)" % argtype) return self if argtype.is_pyobject: - self.error("Cannot take address of Python variable") + self.error("Cannot take address of Python %s" % ( + "variable '%s'" % self.operand.name if self.operand.is_name else + "object attribute '%s'" % self.operand.attribute if self.operand.is_attribute else + "object")) return self if not argtype.is_cpp_class or not self.type: self.type = PyrexTypes.c_ptr_type(argtype) @@ -9787,6 +10621,7 @@ if (self.operand.type.is_cpp_class and self.exception_check == '+'): translate_cpp_exception(code, self.pos, "%s = %s %s;" % (self.result(), self.operator, self.operand.result()), + self.result() if self.type.is_pyobject else None, self.exception_value, self.in_nogil_context) @@ -9864,7 +10699,8 @@ error(self.pos, "Python objects cannot be cast from pointers of primitive types") else: # Should this be an error? - warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.operand.type, self.type)) + warning(self.pos, "No conversion from %s to %s, python object pointer used." % ( + self.operand.type, self.type)) self.operand = self.operand.coerce_to_simple(env) elif from_py and not to_py: if self.type.create_from_py_utility_code(env): @@ -9873,7 +10709,8 @@ if not (self.type.base_type.is_void or self.type.base_type.is_struct): error(self.pos, "Python objects cannot be cast to pointers of primitive types") else: - warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.type, self.operand.type)) + warning(self.pos, "No conversion from %s to %s, python object pointer used." % ( + self.type, self.operand.type)) elif from_py and to_py: if self.typecheck: self.operand = PyTypeTestNode(self.operand, self.type, env, notnone=True) @@ -9885,6 +10722,13 @@ elif self.operand.type.is_fused: self.operand = self.operand.coerce_to(self.type, env) #self.type = self.operand.type + if self.type.is_ptr and self.type.base_type.is_cfunction and self.type.base_type.nogil: + op_type = self.operand.type + if op_type.is_ptr: + op_type = op_type.base_type + if op_type.is_cfunction and not op_type.nogil: + warning(self.pos, + "Casting a GIL-requiring function into a nogil function circumvents GIL validation", 1) return self def is_simple(self): @@ -10091,7 +10935,7 @@ def allocate_temp_result(self, code): if self.temp_code: - raise RuntimeError("temp allocated mulitple times") + raise RuntimeError("temp allocated multiple times") self.temp_code = code.funcstate.allocate_temp(self.type, True) @@ -10099,7 +10943,9 @@ return self.get_cython_array_type(env) def get_cython_array_type(self, env): - return env.global_scope().context.cython_scope.viewscope.lookup("array").type + cython_scope = env.global_scope().context.cython_scope + cython_scope.load_cythonscope() + return cython_scope.viewscope.lookup("array").type def generate_result_code(self, code): from . import Buffer @@ -10121,26 +10967,28 @@ code.putln(code.error_goto(self.operand.pos)) code.putln("}") - code.putln("%s = __pyx_format_from_typeinfo(&%s);" % - (format_temp, type_info)) + code.putln("%s = __pyx_format_from_typeinfo(&%s); %s" % ( + format_temp, + type_info, + code.error_goto_if_null(format_temp, self.pos), + )) + code.put_gotref(format_temp, py_object_type) + buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes) - code.putln('%s = Py_BuildValue((char*) "(" %s ")", %s);' % ( - shapes_temp, buildvalue_fmt, ", ".join(shapes))) + code.putln('%s = Py_BuildValue((char*) "(" %s ")", %s); %s' % ( + shapes_temp, + buildvalue_fmt, + ", ".join(shapes), + code.error_goto_if_null(shapes_temp, self.pos), + )) + code.put_gotref(shapes_temp, py_object_type) - err = "!%s || !%s || !PyBytes_AsString(%s)" % (format_temp, - shapes_temp, - format_temp) - code.putln(code.error_goto_if(err, self.pos)) - code.put_gotref(format_temp) - code.put_gotref(shapes_temp) - - tup = (self.result(), shapes_temp, itemsize, format_temp, - self.mode, self.operand.result()) - code.putln('%s = __pyx_array_new(' - '%s, %s, PyBytes_AS_STRING(%s), ' - '(char *) "%s", (char *) %s);' % tup) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.result()) + code.putln('%s = __pyx_array_new(%s, %s, PyBytes_AS_STRING(%s), (char *) "%s", (char *) %s); %s' % ( + self.result(), + shapes_temp, itemsize, format_temp, self.mode, self.operand.result(), + code.error_goto_if_null(self.result(), self.pos), + )) + self.generate_gotref(code) def dispose(temp): code.put_decref_clear(temp, py_object_type) @@ -10205,9 +11053,7 @@ for attr in path[1:]: operand = AttributeNode(pos=self.pos, obj=operand, attribute=attr) operand = AttributeNode(pos=self.pos, obj=operand, attribute=self.base_type.name) - self.operand = operand - self.__class__ = SizeofVarNode - node = self.analyse_types(env) + node = SizeofVarNode(self.pos, operand=operand).analyse_types(env) return node if self.arg_type is None: base_type = self.base_type.analyse(env) @@ -10251,7 +11097,11 @@ if operand_as_type: self.arg_type = operand_as_type if self.arg_type.is_fused: - self.arg_type = self.arg_type.specialize(env.fused_to_specific) + try: + self.arg_type = self.arg_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.operand.pos, + "Type cannot be specialized since it is not a fused argument to this function") self.__class__ = SizeofTypeNode self.check_type() else: @@ -10272,8 +11122,6 @@ # arg_type ExprNode # is_variable boolean - type = PyrexTypes.error_type - subexprs = ['operand'] arg_type = None @@ -10288,19 +11136,28 @@ typeinfo_entry = typeinfo_module.lookup('type_info') return PyrexTypes.CFakeReferenceType(PyrexTypes.c_const_type(typeinfo_entry.type)) + cpp_message = 'typeid operator' + def analyse_types(self, env): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully + self.cpp_check(env) type_info = self.get_type_info_type(env) if not type_info: self.error("The 'libcpp.typeinfo' module must be cimported to use the typeid() operator") return self + if self.operand is None: + return self # already analysed, no need to repeat self.type = type_info - as_type = self.operand.analyse_as_type(env) + as_type = self.operand.analyse_as_specialized_type(env) if as_type: self.arg_type = as_type self.is_type = True + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation else: self.arg_type = self.operand.analyse_types(env) self.is_type = False + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation if self.arg_type.type.is_pyobject: self.error("Cannot use typeid on a Python object") return self @@ -10331,7 +11188,7 @@ arg_code = self.arg_type.result() translate_cpp_exception(code, self.pos, "%s = typeid(%s);" % (self.temp_code, arg_code), - None, self.in_nogil_context) + None, None, self.in_nogil_context) class TypeofNode(ExprNode): # Compile-time type of an expression, as a string. @@ -10342,16 +11199,20 @@ literal = None type = py_object_type - subexprs = ['literal'] # 'operand' will be ignored after type analysis! + subexprs = ['literal'] # 'operand' will be ignored after type analysis! def analyse_types(self, env): self.operand = self.operand.analyse_types(env) - value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) + value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) literal = StringNode(self.pos, value=value) literal = literal.analyse_types(env) self.literal = literal.coerce_to_pyobject(env) return self + def analyse_as_type(self, env): + self.operand = self.operand.analyse_types(env) + return self.operand.type + def may_be_none(self): return False @@ -10445,7 +11306,7 @@ def infer_type(self, env): return self.result_type(self.operand1.infer_type(env), - self.operand2.infer_type(env)) + self.operand2.infer_type(env), env) def analyse_types(self, env): self.operand1 = self.operand1.analyse_types(env) @@ -10454,10 +11315,15 @@ return self def analyse_operation(self, env): - if self.is_py_operation(): + if self.is_pythran_operation(env): + self.type = self.result_type(self.operand1.type, + self.operand2.type, env) + assert self.type.is_pythran_expr + self.is_temp = 1 + elif self.is_py_operation(): self.coerce_operands_to_pyobjects(env) self.type = self.result_type(self.operand1.type, - self.operand2.type) + self.operand2.type, env) assert self.type.is_pyobject self.is_temp = 1 elif self.is_cpp_operation(): @@ -10471,6 +11337,15 @@ def is_py_operation_types(self, type1, type2): return type1.is_pyobject or type2.is_pyobject or type1.is_ctuple or type2.is_ctuple + def is_pythran_operation(self, env): + return self.is_pythran_operation_types(self.operand1.type, self.operand2.type, env) + + def is_pythran_operation_types(self, type1, type2, env): + # Support only expr op supported_type, or supported_type op expr + return has_np_pythran(env) and \ + (is_pythran_supported_operation_type(type1) and is_pythran_supported_operation_type(type2)) and \ + (is_pythran_expr(type1) or is_pythran_expr(type2)) + def is_cpp_operation(self): return (self.operand1.type.is_cpp_class or self.operand2.type.is_cpp_class) @@ -10487,7 +11362,7 @@ # Used by NumBinopNodes to break up expressions involving multiple # operators so that exceptions can be handled properly. self.is_temp = 1 - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if func_type.is_ptr: func_type = func_type.base_type @@ -10498,7 +11373,9 @@ self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env) self.type = func_type.return_type - def result_type(self, type1, type2): + def result_type(self, type1, type2, env): + if self.is_pythran_operation_types(type1, type2, env): + return PythranExpr(pythran_binop_type(self.operator, type1, type2)) if self.is_py_operation_types(type1, type2): if type2.is_string: type2 = Builtin.bytes_type @@ -10516,6 +11393,8 @@ if result_type is not None: return result_type return py_object_type + elif type1.is_error or type2.is_error: + return PyrexTypes.error_type else: return self.compute_c_result_type(type1, type2) @@ -10538,8 +11417,23 @@ self.operand1.is_ephemeral() or self.operand2.is_ephemeral()) def generate_result_code(self, code): - #print "BinopNode.generate_result_code:", self.operand1, self.operand2 ### - if self.operand1.type.is_pyobject: + if self.type.is_pythran_expr: + code.putln("// Pythran binop") + code.putln("__Pyx_call_destructor(%s);" % self.result()) + if self.operator == '**': + code.putln("new (&%s) decltype(%s){pythonic::numpy::functor::power{}(%s, %s)};" % ( + self.result(), + self.result(), + self.operand1.pythran_result(), + self.operand2.pythran_result())) + else: + code.putln("new (&%s) decltype(%s){%s %s %s};" % ( + self.result(), + self.result(), + self.operand1.pythran_result(), + self.operator, + self.operand2.pythran_result())) + elif self.operand1.type.is_pyobject: function = self.py_operation_function(code) if self.operator == '**': extra_args = ", Py_None" @@ -10553,13 +11447,14 @@ self.operand2.py_result(), extra_args, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.is_temp: # C++ overloaded operators with exception values are currently all # handled through temporaries. if self.is_cpp_operation() and self.exception_check == '+': translate_cpp_exception(code, self.pos, "%s = %s;" % (self.result(), self.calculate_result_code()), + self.result() if self.type.is_pyobject else None, self.exception_value, self.in_nogil_context) else: code.putln("%s = %s;" % (self.result(), self.calculate_result_code())) @@ -10594,9 +11489,8 @@ cpp_type = None if type1.is_cpp_class or type1.is_ptr: cpp_type = type1.find_cpp_operation_type(self.operator, type2) - # FIXME: handle the reversed case? - #if cpp_type is None and (type2.is_cpp_class or type2.is_ptr): - # cpp_type = type2.find_cpp_operation_type(self.operator, type1) + if cpp_type is None and (type2.is_cpp_class or type2.is_ptr): + cpp_type = type2.find_cpp_operation_type(self.operator, type1) # FIXME: do we need to handle other cases here? return cpp_type @@ -10677,8 +11571,8 @@ def c_types_okay(self, type1, type2): #print "NumBinopNode.c_types_okay:", type1, type2 ### - return (type1.is_numeric or type1.is_enum) \ - and (type2.is_numeric or type2.is_enum) + return (type1.is_numeric or type1.is_enum) \ + and (type2.is_numeric or type2.is_enum) def generate_evaluation_code(self, code): if self.overflow_check: @@ -10701,10 +11595,11 @@ self.operand2.result(), self.overflow_bit_node.overflow_bit) elif self.type.is_cpp_class or self.infix: - return "(%s %s %s)" % ( - self.operand1.result(), - self.operator, - self.operand2.result()) + if is_pythran_expr(self.type): + result1, result2 = self.operand1.pythran_result(), self.operand2.pythran_result() + else: + result1, result2 = self.operand1.result(), self.operand2.result() + return "(%s %s %s)" % (result1, self.operator, result2) else: func = self.type.binary_op(self.operator) if func is None: @@ -10770,7 +11665,7 @@ def infer_builtin_types_operation(self, type1, type2): # b'abc' + 'abc' raises an exception in Py3, # so we can safely infer the Py2 type for bytes here - string_types = (bytes_type, str_type, basestring_type, unicode_type) + string_types = (bytes_type, bytearray_type, str_type, basestring_type, unicode_type) if type1 in string_types and type2 in string_types: return string_types[max(string_types.index(type1), string_types.index(type2))] @@ -10787,19 +11682,36 @@ self, type1, type2) def py_operation_function(self, code): - is_unicode_concat = False - if isinstance(self.operand1, FormattedValueNode) or isinstance(self.operand2, FormattedValueNode): - is_unicode_concat = True - else: - type1, type2 = self.operand1.type, self.operand2.type - if type1 is unicode_type or type2 is unicode_type: - is_unicode_concat = type1.is_builtin_type and type2.is_builtin_type + type1, type2 = self.operand1.type, self.operand2.type + func = None + if type1 is unicode_type or type2 is unicode_type: + if type1 in (unicode_type, str_type) and type2 in (unicode_type, str_type): + is_unicode_concat = True + elif isinstance(self.operand1, FormattedValueNode) or isinstance(self.operand2, FormattedValueNode): + # Assume that even if we don't know the second type, it's going to be a string. + is_unicode_concat = True + else: + # Operation depends on the second type. + is_unicode_concat = False + + if is_unicode_concat: + if self.inplace or self.operand1.is_temp: + code.globalstate.use_utility_code( + UtilityCode.load_cached("UnicodeConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyUnicode_Concat' + elif type1 is str_type and type2 is str_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("StrConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyStr_Concat' - if is_unicode_concat: + if func: + # any necessary utility code will be got by "NumberAdd" in generate_evaluation_code + if self.inplace or self.operand1.is_temp: + func += 'InPlace' # upper case to indicate unintuitive macro if self.operand1.may_be_none() or self.operand2.may_be_none(): - return '__Pyx_PyUnicode_ConcatSafe' - else: - return '__Pyx_PyUnicode_Concat' + func += 'Safe' + return func + return super(AddNode, self).py_operation_function(code) @@ -10819,6 +11731,24 @@ class MulNode(NumBinopNode): # '*' operator. + def analyse_types(self, env): + # TODO: we could also optimise the case of "[...] * 2 * n", i.e. with an existing 'mult_factor' + if self.operand1.is_sequence_constructor and self.operand1.mult_factor is None: + operand2 = self.operand2.analyse_types(env) + if operand2.type.is_int or operand2.type is long_type: + return self.analyse_sequence_mul(env, self.operand1, operand2) + elif self.operand2.is_sequence_constructor and self.operand2.mult_factor is None: + operand1 = self.operand1.analyse_types(env) + if operand1.type.is_int or operand1.type is long_type: + return self.analyse_sequence_mul(env, self.operand2, operand1) + + return NumBinopNode.analyse_types(self, env) + + def analyse_sequence_mul(self, env, seq, mult): + assert seq.mult_factor is None + seq.mult_factor = mult + return seq.analyse_types(env) + def is_py_operation_types(self, type1, type2): if ((type1.is_string and type2.is_int) or (type2.is_string and type1.is_int)): @@ -10829,7 +11759,7 @@ def infer_builtin_types_operation(self, type1, type2): # let's assume that whatever builtin type you multiply a string with # will either return a string of the same type or fail with an exception - string_types = (bytes_type, str_type, basestring_type, unicode_type) + string_types = (bytes_type, bytearray_type, str_type, basestring_type, unicode_type) if type1 in string_types and type2.is_builtin_type: return type1 if type2 in string_types and type1.is_builtin_type: @@ -10899,7 +11829,7 @@ self._check_truedivision(env) return self.result_type( self.operand1.infer_type(env), - self.operand2.infer_type(env)) + self.operand2.infer_type(env), env) def analyse_operation(self, env): self._check_truedivision(env) @@ -10917,7 +11847,7 @@ self.operand2 = self.operand2.coerce_to_simple(env) def compute_c_result_type(self, type1, type2): - if self.operator == '/' and self.ctruedivision: + if self.operator == '/' and self.ctruedivision and not type1.is_cpp_class and not type2.is_cpp_class: if not type1.is_float and not type2.is_float: widest_type = PyrexTypes.widest_numeric_type(type1, PyrexTypes.c_double_type) widest_type = PyrexTypes.widest_numeric_type(type2, widest_type) @@ -10933,9 +11863,11 @@ def generate_evaluation_code(self, code): if not self.type.is_pyobject and not self.type.is_complex: if self.cdivision is None: - self.cdivision = (code.globalstate.directives['cdivision'] - or not self.type.signed - or self.type.is_float) + self.cdivision = ( + code.globalstate.directives['cdivision'] + or self.type.is_float + or ((self.type.is_numeric or self.type.is_enum) and not self.type.signed) + ) if not self.cdivision: code.globalstate.use_utility_code( UtilityCode.load_cached("DivInt", "CMath.c").specialize(self.type)) @@ -10968,7 +11900,7 @@ minus1_check = '(!(((%s)-1) > 0)) && unlikely(%s == (%s)-1)' % ( type_of_op2, self.operand2.result(), type_of_op2) code.putln("else if (sizeof(%s) == sizeof(long) && %s " - " && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( + " && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( self.type.empty_declaration_code(), minus1_check, self.operand1.result())) @@ -11006,7 +11938,7 @@ code.putln("}") def calculate_result_code(self): - if self.type.is_complex: + if self.type.is_complex or self.is_cpp_operation(): return NumBinopNode.calculate_result_code(self) elif self.type.is_float and self.operator == '//': return "floor(%s / %s)" % ( @@ -11028,6 +11960,20 @@ self.operand2.result()) +_find_formatting_types = re.compile( + br"%" + br"(?:%|" # %% + br"(?:\([^)]+\))?" # %(name) + br"[-+#,0-9 ]*([a-z])" # %.2f etc. + br")").findall + +# These format conversion types can never trigger a Unicode string conversion in Py2. +_safe_bytes_formats = frozenset({ + # Excludes 's' and 'r', which can generate non-bytes strings. + b'd', b'i', b'o', b'u', b'x', b'X', b'e', b'E', b'f', b'F', b'g', b'G', b'c', b'b', b'a', +}) + + class ModNode(DivNode): # '%' operator. @@ -11037,7 +11983,7 @@ or NumBinopNode.is_py_operation_types(self, type1, type2)) def infer_builtin_types_operation(self, type1, type2): - # b'%s' % xyz raises an exception in Py3, so it's safe to infer the type for Py2 + # b'%s' % xyz raises an exception in Py3<3.5, so it's safe to infer the type for Py2 and later Py3's. if type1 is unicode_type: # None + xyz may be implemented by RHS if type2.is_builtin_type or not self.operand1.may_be_none(): @@ -11047,6 +11993,11 @@ return type2 elif type2.is_numeric: return type1 + elif self.operand1.is_string_literal: + if type1 is str_type or type1 is bytes_type: + if set(_find_formatting_types(self.operand1.value)) <= _safe_bytes_formats: + return type1 + return basestring_type elif type1 is bytes_type and not type2.is_builtin_type: return None # RHS might implement '% operator differently in Py3 else: @@ -11098,13 +12049,19 @@ self.operand2.result()) def py_operation_function(self, code): - if self.operand1.type is unicode_type: - if self.operand1.may_be_none(): + type1, type2 = self.operand1.type, self.operand2.type + # ("..." % x) must call "x.__rmod__()" for string subtypes. + if type1 is unicode_type: + if self.operand1.may_be_none() or ( + type2.is_extension_type and type2.subtype_of(type1) or + type2 is py_object_type and not isinstance(self.operand2, CoerceToPyTypeNode)): return '__Pyx_PyUnicode_FormatSafe' else: return 'PyUnicode_Format' - elif self.operand1.type is str_type: - if self.operand1.may_be_none(): + elif type1 is str_type: + if self.operand1.may_be_none() or ( + type2.is_extension_type and type2.subtype_of(type1) or + type2 is py_object_type and not isinstance(self.operand2, CoerceToPyTypeNode)): return '__Pyx_PyString_FormatSafe' else: return '__Pyx_PyString_Format' @@ -11137,6 +12094,12 @@ error(self.pos, "got unexpected types for C power operator: %s, %s" % (self.operand1.type, self.operand2.type)) + def compute_c_result_type(self, type1, type2): + c_result_type = super(PowNode, self).compute_c_result_type(type1, type2) + if isinstance(self.operand2.constant_result, _py_int_types) and self.operand2.constant_result < 0: + c_result_type = PyrexTypes.widest_numeric_type(c_result_type, PyrexTypes.c_double_type) + return c_result_type + def calculate_result_code(self): # Work around MSVC overloading ambiguity. def typecast(operand): @@ -11245,7 +12208,7 @@ operator=self.operator, operand1=operand1, operand2=operand2) - def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label, fall_through): + def generate_bool_evaluation_code(self, code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through): code.mark_pos(self.pos) outer_labels = (and_label, or_label) @@ -11254,19 +12217,20 @@ else: my_label = or_label = code.new_label('next_or') self.operand1.generate_bool_evaluation_code( - code, final_result_temp, and_label, or_label, end_label, my_label) + code, final_result_temp, final_result_type, and_label, or_label, end_label, my_label) and_label, or_label = outer_labels code.put_label(my_label) self.operand2.generate_bool_evaluation_code( - code, final_result_temp, and_label, or_label, end_label, fall_through) + code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through) def generate_evaluation_code(self, code): self.allocate_temp_result(code) + result_type = PyrexTypes.py_object_type if self.type.is_pyobject else self.type or_label = and_label = None end_label = code.new_label('bool_binop_done') - self.generate_bool_evaluation_code(code, self.result(), and_label, or_label, end_label, end_label) + self.generate_bool_evaluation_code(code, self.result(), result_type, and_label, or_label, end_label, end_label) code.put_label(end_label) gil_message = "Truth-testing Python object" @@ -11351,7 +12315,7 @@ test_result = self.arg.result() return (test_result, self.arg.type.is_pyobject) - def generate_bool_evaluation_code(self, code, final_result_temp, and_label, or_label, end_label, fall_through): + def generate_bool_evaluation_code(self, code, final_result_temp, final_result_type, and_label, or_label, end_label, fall_through): code.mark_pos(self.pos) # x => x @@ -11394,7 +12358,7 @@ code.putln("} else {") self.value.generate_evaluation_code(code) self.value.make_owned_reference(code) - code.putln("%s = %s;" % (final_result_temp, self.value.result())) + code.putln("%s = %s;" % (final_result_temp, self.value.result_as(final_result_type))) self.value.generate_post_assignment_code(code) # disposal: {not (and_label and or_label) [else]} self.arg.generate_disposal_code(code) @@ -11406,6 +12370,9 @@ code.putln("}") self.arg.free_temps(code) + def analyse_types(self, env): + return self + class CondExprNode(ExprNode): # Short-circuiting conditional expression. @@ -11416,6 +12383,7 @@ true_val = None false_val = None + is_temp = True subexprs = ['test', 'true_val', 'false_val'] @@ -11440,7 +12408,6 @@ self.test = self.test.analyse_types(env).coerce_to_boolean(env) self.true_val = self.true_val.analyse_types(env) self.false_val = self.false_val.analyse_types(env) - self.is_temp = 1 return self.analyse_result_type(env) def analyse_result_type(self, env): @@ -11614,22 +12581,22 @@ new_common_type = None # catch general errors - if type1 == str_type and (type2.is_string or type2 in (bytes_type, unicode_type)) or \ - type2 == str_type and (type1.is_string or type1 in (bytes_type, unicode_type)): + if (type1 == str_type and (type2.is_string or type2 in (bytes_type, unicode_type)) or + type2 == str_type and (type1.is_string or type1 in (bytes_type, unicode_type))): error(self.pos, "Comparisons between bytes/unicode and str are not portable to Python 3") new_common_type = error_type # try to use numeric comparisons where possible elif type1.is_complex or type2.is_complex: - if op not in ('==', '!=') \ - and (type1.is_complex or type1.is_numeric) \ - and (type2.is_complex or type2.is_numeric): + if (op not in ('==', '!=') + and (type1.is_complex or type1.is_numeric) + and (type2.is_complex or type2.is_numeric)): error(self.pos, "complex types are unordered") new_common_type = error_type elif type1.is_pyobject: - new_common_type = type1 + new_common_type = Builtin.complex_type if type1.subtype_of(Builtin.complex_type) else py_object_type elif type2.is_pyobject: - new_common_type = type2 + new_common_type = Builtin.complex_type if type2.subtype_of(Builtin.complex_type) else py_object_type else: new_common_type = PyrexTypes.widest_numeric_type(type1, type2) elif type1.is_numeric and type2.is_numeric: @@ -11755,6 +12722,11 @@ self.special_bool_cmp_utility_code = UtilityCode.load_cached("PyDictContains", "ObjectHandling.c") self.special_bool_cmp_function = "__Pyx_PyDict_ContainsTF" return True + elif self.operand2.type is Builtin.set_type: + self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable") + self.special_bool_cmp_utility_code = UtilityCode.load_cached("PySetContains", "ObjectHandling.c") + self.special_bool_cmp_function = "__Pyx_PySet_ContainsTF" + return True elif self.operand2.type is Builtin.unicode_type: self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable") self.special_bool_cmp_utility_code = UtilityCode.load_cached("PyUnicodeContains", "StringTools.c") @@ -11842,8 +12814,15 @@ self.c_operator(op), code2) if self.is_cpp_comparison() and self.exception_check == '+': - translate_cpp_exception(code, self.pos, statement, self.exception_value, self.in_nogil_context) - code.putln(statement) + translate_cpp_exception( + code, + self.pos, + statement, + result_code if self.type.is_pyobject else None, + self.exception_value, + self.in_nogil_context) + else: + code.putln(statement) def c_operator(self, op): if op == 'is': @@ -11874,7 +12853,14 @@ is_memslice_nonecheck = False def infer_type(self, env): - # TODO: Actually implement this (after merging with -unstable). + type1 = self.operand1.infer_type(env) + type2 = self.operand2.infer_type(env) + + if is_pythran_expr(type1) or is_pythran_expr(type2): + if is_pythran_supported_type(type1) and is_pythran_supported_type(type2): + return PythranExpr(pythran_binop_type(self.operator, type1, type2)) + + # TODO: implement this for other types. return py_object_type def type_dependencies(self, env): @@ -11897,6 +12883,14 @@ error(self.pos, "Cascading comparison not yet supported for cpp types.") return self + type1 = self.operand1.type + type2 = self.operand2.type + if is_pythran_expr(type1) or is_pythran_expr(type2): + if is_pythran_supported_type(type1) and is_pythran_supported_type(type2): + self.type = PythranExpr(pythran_binop_type(self.operator, type1, type2)) + self.is_pycmp = False + return self + if self.analyse_memoryviewslice_comparison(env): return self @@ -11929,16 +12923,16 @@ elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = py_object_type self.is_pycmp = True elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = self.find_common_type(env, self.operator, self.operand1) self.is_pycmp = common_type.is_pyobject @@ -11985,7 +12979,7 @@ self.exception_value = func_type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if len(func_type.args) == 1: self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env) @@ -12036,18 +13030,19 @@ return self.operand1.check_const() and self.operand2.check_const() def calculate_result_code(self): - if self.operand1.type.is_complex: + operand1, operand2 = self.operand1, self.operand2 + if operand1.type.is_complex: if self.operator == "!=": negation = "!" else: negation = "" return "(%s%s(%s, %s))" % ( negation, - self.operand1.type.binary_op('=='), - self.operand1.result(), - self.operand2.result()) + operand1.type.binary_op('=='), + operand1.result(), + operand2.result()) elif self.is_c_string_contains(): - if self.operand2.type is unicode_type: + if operand2.type is unicode_type: method = "__Pyx_UnicodeContainsUCS4" else: method = "__Pyx_BytesContains" @@ -12058,16 +13053,18 @@ return "(%s%s(%s, %s))" % ( negation, method, - self.operand2.result(), - self.operand1.result()) + operand2.result(), + operand1.result()) else: - result1 = self.operand1.result() - result2 = self.operand2.result() - if self.is_memslice_nonecheck: - if self.operand1.type.is_memoryviewslice: - result1 = "((PyObject *) %s.memview)" % result1 - else: - result2 = "((PyObject *) %s.memview)" % result2 + if is_pythran_expr(self.type): + result1, result2 = operand1.pythran_result(), operand2.pythran_result() + else: + result1, result2 = operand1.result(), operand2.result() + if self.is_memslice_nonecheck: + if operand1.type.is_memoryviewslice: + result1 = "((PyObject *) %s.memview)" % result1 + else: + result2 = "((PyObject *) %s.memview)" % result2 return "(%s %s %s)" % ( result1, @@ -12123,7 +13120,7 @@ cascade = None coerced_operand2 = None - constant_result = constant_value_not_set # FIXME: where to calculate this? + constant_result = constant_value_not_set # FIXME: where to calculate this? def infer_type(self, env): # TODO: Actually implement this (after merging with -unstable). @@ -12270,6 +13267,9 @@ code.annotate((file, line, col-1), AnnotationItem( style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type))) + def analyse_types(self, env): + return self + class CoerceToMemViewSliceNode(CoercionNode): """ @@ -12283,18 +13283,17 @@ CoercionNode.__init__(self, arg) self.type = dst_type self.is_temp = 1 - self.env = env self.use_managed_ref = True self.arg = arg + self.type.create_from_py_utility_code(env) def generate_result_code(self, code): - self.type.create_from_py_utility_code(self.env) - code.putln("%s = %s(%s);" % (self.result(), - self.type.from_py_function, - self.arg.py_result())) - - error_cond = self.type.error_condition(self.result()) - code.putln(code.error_goto_if(error_cond, self.pos)) + code.putln(self.type.from_py_call_code( + self.arg.py_result(), + self.result(), + self.pos, + code + )) class CastNode(CoercionNode): @@ -12322,9 +13321,10 @@ exact_builtin_type = True def __init__(self, arg, dst_type, env, notnone=False): - # The arg is know to be a Python object, and + # The arg is known to be a Python object, and # the dst_type is known to be an extension type. - assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type" + assert dst_type.is_extension_type or dst_type.is_builtin_type, \ + "PyTypeTest for %s against non extension type %s" % (arg.type, dst_type) CoercionNode.__init__(self, arg) self.type = dst_type self.result_ctype = arg.ctype() @@ -12353,6 +13353,15 @@ def nonlocally_immutable(self): return self.arg.nonlocally_immutable() + def reanalyse(self): + if self.type != self.arg.type or not self.arg.is_temp: + return self + if not self.type.typeobj_is_available(): + return self + if self.arg.may_be_none() and self.notnone: + return self.arg.as_none_safe_node("Cannot convert NoneType to %.200s" % self.type.name) + return self.arg + def calculate_constant_result(self): # FIXME pass @@ -12366,6 +13375,8 @@ type_test = self.type.type_test_code( self.arg.py_result(), self.notnone, exact=self.exact_builtin_type) + code.globalstate.use_utility_code(UtilityCode.load_cached( + "RaiseUnexpectedTypeError", "ObjectHandling.c")) else: type_test = self.type.type_test_code( self.arg.py_result(), self.notnone) @@ -12380,9 +13391,18 @@ def generate_post_assignment_code(self, code): self.arg.generate_post_assignment_code(code) + def allocate_temp_result(self, code): + pass + + def release_temp_result(self, code): + pass + def free_temps(self, code): self.arg.free_temps(code) + def free_subexpr_temps(self, code): + self.arg.free_subexpr_temps(code) + class NoneCheckNode(CoercionNode): # This node is used to check that a Python object is not None and @@ -12392,7 +13412,7 @@ is_nonecheck = True def __init__(self, arg, exception_type_cname, exception_message, - exception_format_args): + exception_format_args=()): CoercionNode.__init__(self, arg) self.type = arg.type self.result_ctype = arg.ctype() @@ -12400,7 +13420,7 @@ self.exception_message = exception_message self.exception_format_args = tuple(exception_format_args or ()) - nogil_check = None # this node only guards an operation that would fail already + nogil_check = None # this node only guards an operation that would fail already def analyse_types(self, env): return self @@ -12428,6 +13448,19 @@ else: raise Exception("unsupported type") + @classmethod + def generate(cls, arg, code, exception_message, + exception_type_cname="PyExc_TypeError", exception_format_args=(), in_nogil_context=False): + node = cls(arg, exception_type_cname, exception_message, exception_format_args) + node.in_nogil_context = in_nogil_context + node.put_nonecheck(code) + + @classmethod + def generate_if_needed(cls, arg, code, exception_message, + exception_type_cname="PyExc_TypeError", exception_format_args=(), in_nogil_context=False): + if arg.may_be_none(): + cls.generate(arg, code, exception_message, exception_type_cname, exception_format_args, in_nogil_context) + def put_nonecheck(self, code): code.putln( "if (unlikely(%s == Py_None)) {" % self.condition()) @@ -12510,7 +13543,7 @@ def coerce_to_boolean(self, env): arg_type = self.arg.type if (arg_type == PyrexTypes.c_bint_type or - (arg_type.is_pyobject and arg_type.name == 'bool')): + (arg_type.is_pyobject and arg_type.name == 'bool')): return self.arg.coerce_to_temp(env) else: return CoerceToBooleanNode(self, env) @@ -12534,7 +13567,7 @@ self.target_type), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceIntToBytesNode(CoerceToPyTypeNode): @@ -12574,7 +13607,7 @@ code.error_goto_if_null(self.result(), self.pos))) if temp is not None: code.funcstate.release_temp(temp) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceFromPyTypeNode(CoercionNode): @@ -12602,10 +13635,17 @@ return (self.type.is_ptr and not self.type.is_array) and self.arg.is_ephemeral() def generate_result_code(self, code): + from_py_function = None + # for certain source types, we can do better than the generic coercion + if self.type.is_string and self.arg.type is bytes_type: + if self.type.from_py_function.startswith('__Pyx_PyObject_As'): + from_py_function = '__Pyx_PyBytes' + self.type.from_py_function[len('__Pyx_PyObject'):] + NoneCheckNode.generate_if_needed(self.arg, code, "expected bytes, NoneType found") + code.putln(self.type.from_py_call_code( - self.arg.py_result(), self.result(), self.pos, code)) + self.arg.py_result(), self.result(), self.pos, code, from_py_function=from_py_function)) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def nogil_check(self, env): error(self.pos, "Coercion from Python not allowed without the GIL") @@ -12623,6 +13663,7 @@ Builtin.set_type: 'PySet_GET_SIZE', Builtin.frozenset_type: 'PySet_GET_SIZE', Builtin.bytes_type: 'PyBytes_GET_SIZE', + Builtin.bytearray_type: 'PyByteArray_GET_SIZE', Builtin.unicode_type: '__Pyx_PyUnicode_IS_TRUE', } @@ -12651,11 +13692,9 @@ return test_func = self._special_builtins.get(self.arg.type) if test_func is not None: - code.putln("%s = (%s != Py_None) && (%s(%s) != 0);" % ( - self.result(), - self.arg.py_result(), - test_func, - self.arg.py_result())) + checks = ["(%s != Py_None)" % self.arg.py_result()] if self.arg.may_be_none() else [] + checks.append("(%s(%s) != 0)" % (test_func, self.arg.py_result())) + code.putln("%s = %s;" % (self.result(), '&&'.join(checks))) else: code.putln( "%s = __Pyx_PyObject_IsTrue(%s); %s" % ( @@ -12663,6 +13702,9 @@ self.arg.py_result(), code.error_goto_if_neg(self.result(), self.pos))) + def analyse_types(self, env): + return self + class CoerceToComplexNode(CoercionNode): @@ -12688,6 +13730,9 @@ def generate_result_code(self, code): pass + def analyse_types(self, env): + return self + class CoerceToTempNode(CoercionNode): # This node is used to force the result of another node # to be stored in a temporary. It is only used if the @@ -12707,6 +13752,9 @@ # The arg is always already analysed return self + def may_be_none(self): + return self.arg.may_be_none() + def coerce_to_boolean(self, env): self.arg = self.arg.coerce_to_boolean(env) if self.arg.is_simple(): @@ -12721,11 +13769,12 @@ code.putln("%s = %s;" % ( self.result(), self.arg.result_as(self.ctype()))) if self.use_managed_ref: - if self.type.is_pyobject: + if not self.type.is_memoryviewslice: code.put_incref(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_incref_memoryviewslice(self.result(), - not self.in_nogil_context) + else: + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) + class ProxyNode(CoercionNode): """ @@ -12741,22 +13790,24 @@ def __init__(self, arg): super(ProxyNode, self).__init__(arg) self.constant_result = arg.constant_result - self._proxy_type() + self.update_type_and_entry() def analyse_types(self, env): self.arg = self.arg.analyse_expressions(env) - self._proxy_type() + self.update_type_and_entry() return self def infer_type(self, env): return self.arg.infer_type(env) - def _proxy_type(self): - if hasattr(self.arg, 'type'): - self.type = self.arg.type + def update_type_and_entry(self): + type = getattr(self.arg, 'type', None) + if type: + self.type = type self.result_ctype = self.arg.result_ctype - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def generate_result_code(self, code): self.arg.generate_result_code(code) @@ -12787,17 +13838,19 @@ # disposal code for it. The original owner of the argument # node is responsible for doing those things. - subexprs = [] # Arg is not considered a subexpr + subexprs = [] # Arg is not considered a subexpr nogil_check = None def __init__(self, arg): CoercionNode.__init__(self, arg) self.constant_result = arg.constant_result - if hasattr(arg, 'type'): - self.type = arg.type + type = getattr(arg, 'type', None) + if type: + self.type = type self.result_ctype = arg.result_ctype - if hasattr(arg, 'entry'): - self.entry = arg.entry + arg_entry = getattr(arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def result(self): return self.arg.result() @@ -12815,8 +13868,9 @@ self.type = self.arg.type self.result_ctype = self.arg.result_ctype self.is_temp = 1 - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry return self def coerce_to(self, dest_type, env): @@ -12825,7 +13879,7 @@ return super(CloneNode, self).coerce_to(dest_type, env) def is_simple(self): - return True # result is always in a temp (or a name) + return True # result is always in a temp (or a name) def generate_evaluation_code(self, code): pass @@ -12836,10 +13890,38 @@ def generate_disposal_code(self, code): pass + def generate_post_assignment_code(self, code): + # if we're assigning from a CloneNode then it's "giveref"ed away, so it does + # need a matching incref (ideally this should happen before the assignment though) + if self.is_temp: # should usually be true + code.put_incref(self.result(), self.ctype()) + def free_temps(self, code): pass +class CppOptionalTempCoercion(CoercionNode): + """ + Used only in CoerceCppTemps - handles cases the temp is actually a OptionalCppClassType (and thus needs dereferencing when on the rhs) + """ + is_temp = False + + @property + def type(self): + return self.arg.type + + def calculate_result_code(self): + return "(*%s)" % self.arg.result() + + def generate_result_code(self, code): + pass + + def _make_move_result_rhs(self, result, optional=False): + # this wouldn't normally get moved (because it isn't a temp), but force it to be because it + # is a thin wrapper around a temp + return super(CppOptionalTempCoercion, self)._make_move_result_rhs(result, optional=False) + + class CMethodSelfCloneNode(CloneNode): # Special CloneNode for the self argument of builtin C methods # that accepts subtypes of the builtin type. This is safe only @@ -12891,77 +13973,193 @@ self.result(), self.body.result(), code.intern_identifier(StringEncoding.EncodedString("__doc__")), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) +class AnnotationNode(ExprNode): + # Deals with the two possible uses of an annotation. + # 1. The post PEP-563 use where an annotation is stored + # as a string + # 2. The Cython use where the annotation can indicate an + # object type + # + # Doesn't handle the pre PEP-563 version where the + # annotation is evaluated into a Python Object. + subexprs = [] -#------------------------------------------------------------------------------------ -# -# Runtime support code -# -#------------------------------------------------------------------------------------ + # 'untyped' is set for fused specializations: + # Once a fused function has been created we don't want + # annotations to override an already set type. + untyped = False -pyerr_occurred_withgil_utility_code= UtilityCode( -proto = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ -""", -impl = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { - int err; - #ifdef WITH_THREAD - PyGILState_STATE _save = PyGILState_Ensure(); - #endif - err = !!PyErr_Occurred(); - #ifdef WITH_THREAD - PyGILState_Release(_save); - #endif - return err; -} -""" -) + def __init__(self, pos, expr, string=None): + """string is expected to already be a StringNode or None""" + ExprNode.__init__(self, pos) + if string is None: + # import doesn't work at top of file? + from .AutoDocTransforms import AnnotationWriter + string = StringEncoding.EncodedString( + AnnotationWriter(description="annotation").write(expr)) + string = StringNode(pos, unicode_value=string, value=string.as_utf8_string()) + self.string = string + self.expr = expr -#------------------------------------------------------------------------------------ + def analyse_types(self, env): + return self # nothing needs doing -raise_unbound_local_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { - PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); -} -""") + def analyse_as_type(self, env): + # for compatibility when used as a return_type_node, have this interface too + return self.analyse_type_annotation(env)[1] -raise_closure_name_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { - PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); -} -""") + def analyse_type_annotation(self, env, assigned_value=None): + if self.untyped: + # Already applied as a fused type, not re-evaluating it here. + return None, None + annotation = self.expr + base_type = None + is_ambiguous = False + explicit_pytype = explicit_ctype = False + if annotation.is_dict_literal: + warning(annotation.pos, + "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.", level=1) + for name, value in annotation.key_value_pairs: + if not name.is_string_literal: + continue + if name.value in ('type', b'type'): + explicit_pytype = True + if not explicit_ctype: + annotation = value + elif name.value in ('ctype', b'ctype'): + explicit_ctype = True + annotation = value + if explicit_pytype and explicit_ctype: + warning(annotation.pos, "Duplicate type declarations found in signature annotation", level=1) + arg_type = annotation.analyse_as_type(env) + if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): + # Map builtin numeric Python types to C types in safe cases. + if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: + assigned_type = assigned_value.infer_type(env) + if assigned_type and assigned_type.is_pyobject: + # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type + is_ambiguous = True + arg_type = None + # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations + if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): + arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type + elif arg_type is not None and annotation.is_string_literal: + warning(annotation.pos, + "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.", + level=1) + elif arg_type is not None and arg_type.is_complex: + # creating utility code needs to be special-cased for complex types + arg_type.create_declaration_utility_code(env) + if arg_type is not None: + if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: + warning(annotation.pos, + "Python type declaration in signature annotation does not refer to a Python type") + base_type = Nodes.CAnalysedBaseTypeNode( + annotation.pos, type=arg_type, is_arg=True) + elif is_ambiguous: + warning(annotation.pos, "Ambiguous types in annotation, ignoring") + else: + warning(annotation.pos, "Unknown type declaration in annotation, ignoring") + return base_type, arg_type -# Don't inline the function, it should really never be called in production -raise_unbound_memoryview_utility_code_nogil = UtilityCode( -proto = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname); -""", -impl = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { - #ifdef WITH_THREAD - PyGILState_STATE gilstate = PyGILState_Ensure(); - #endif - __Pyx_RaiseUnboundLocalError(varname); - #ifdef WITH_THREAD - PyGILState_Release(gilstate); - #endif -} -""", -requires = [raise_unbound_local_error_utility_code]) -#------------------------------------------------------------------------------------ +class AssignmentExpressionNode(ExprNode): + """ + Also known as a named expression or the walrus operator + + Arguments + lhs - NameNode - not stored directly as an attribute of the node + rhs - ExprNode + + Attributes + rhs - ExprNode + assignment - SingleAssignmentNode + """ + # subexprs and child_attrs are intentionally different here, because the assignment is not an expression + subexprs = ["rhs"] + child_attrs = ["rhs", "assignment"] # This order is important for control-flow (i.e. xdecref) to be right + + is_temp = False + assignment = None + clone_node = None + + def __init__(self, pos, lhs, rhs, **kwds): + super(AssignmentExpressionNode, self).__init__(pos, **kwds) + self.rhs = ProxyNode(rhs) + assign_expr_rhs = CloneNode(self.rhs) + self.assignment = SingleAssignmentNode( + pos, lhs=lhs, rhs=assign_expr_rhs, is_assignment_expression=True) + + @property + def type(self): + return self.rhs.type + + @property + def target_name(self): + return self.assignment.lhs.name + + def infer_type(self, env): + return self.rhs.infer_type(env) + + def analyse_declarations(self, env): + self.assignment.analyse_declarations(env) + + def analyse_types(self, env): + # we're trying to generate code that looks roughly like: + # __pyx_t_1 = rhs + # lhs = __pyx_t_1 + # __pyx_t_1 + # (plus any reference counting that's needed) + + self.rhs = self.rhs.analyse_types(env) + if not self.rhs.arg.is_temp: + if not self.rhs.arg.is_literal: + # for anything but the simplest cases (where it can be used directly) + # we convert rhs to a temp, because CloneNode requires arg to be a temp + self.rhs.arg = self.rhs.arg.coerce_to_temp(env) + else: + # For literals we can optimize by just using the literal twice + # + # We aren't including `self.rhs.is_name` in this optimization + # because that goes wrong for assignment expressions run in + # parallel. e.g. `(a := b) + (b := a + c)`) + # This is a special case of https://github.com/cython/cython/issues/4146 + # TODO - once that's fixed general revisit this code and possibly + # use coerce_to_simple + self.assignment.rhs = copy.copy(self.rhs) + + # TODO - there's a missed optimization in the code generation stage + # for self.rhs.arg.is_temp: an incref/decref pair can be removed + # (but needs a general mechanism to do that) + self.assignment = self.assignment.analyse_types(env) + return self + + def coerce_to(self, dst_type, env): + if dst_type == self.assignment.rhs.type: + # in this quite common case (for example, when both lhs, and self are being coerced to Python) + # we can optimize the coercion out by sharing it between + # this and the assignment + old_rhs_arg = self.rhs.arg + if isinstance(old_rhs_arg, CoerceToTempNode): + old_rhs_arg = old_rhs_arg.arg + rhs_arg = old_rhs_arg.coerce_to(dst_type, env) + if rhs_arg is not old_rhs_arg: + self.rhs.arg = rhs_arg + self.rhs.update_type_and_entry() + # clean up the old coercion node that the assignment has likely generated + if (isinstance(self.assignment.rhs, CoercionNode) + and not isinstance(self.assignment.rhs, CloneNode)): + self.assignment.rhs = self.assignment.rhs.arg + self.assignment.rhs.type = self.assignment.rhs.arg.type + return self + return super(AssignmentExpressionNode, self).coerce_to(dst_type, env) + + def calculate_result_code(self): + return self.rhs.result() -raise_too_many_values_to_unpack = UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c") -raise_need_more_values_to_unpack = UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c") -tuple_unpacking_error_code = UtilityCode.load_cached("UnpackTupleError", "ObjectHandling.c") + def generate_result_code(self, code): + # we have to do this manually because it isn't a subexpression + self.assignment.generate_execution_code(code) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/FlowControl.pxd cython-0.20.1+1~202203241016-9537/Cython/Compiler/FlowControl.pxd --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/FlowControl.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/FlowControl.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython @@ -11,10 +11,8 @@ cdef public list stats cdef public dict gen cdef public set bounded - cdef public dict input - cdef public dict output - # Big integer it bitsets + # Big integer bitsets cdef public object i_input cdef public object i_output cdef public object i_gen @@ -107,6 +105,7 @@ cdef list stack cdef object env cdef ControlFlow flow + cdef object object_expr cdef bint in_inplace_assignment cpdef mark_assignment(self, lhs, rhs=*) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/FlowControl.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/FlowControl.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/FlowControl.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/FlowControl.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,21 +1,22 @@ +# cython: language_level=3str +# cython: auto_pickle=True + from __future__ import absolute_import import cython -cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, - Builtin=object, InternalError=object, error=object, warning=object, - py_object_type=object, unspecified_type=object, - object_expr=object, fake_rhs_expr=object, TypedExprNode=object) +cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, Builtin=object, + Options=object, TreeVisitor=object, CythonTransform=object, + InternalError=object, error=object, warning=object, + fake_rhs_expr=object, TypedExprNode=object) from . import Builtin from . import ExprNodes from . import Nodes from . import Options -from .PyrexTypes import py_object_type, unspecified_type from . import PyrexTypes from .Visitor import TreeVisitor, CythonTransform from .Errors import error, warning, InternalError -from .Optimize import ConstantFolding class TypedExprNode(ExprNodes.ExprNode): @@ -28,9 +29,8 @@ def may_be_none(self): return self._may_be_none != False -object_expr = TypedExprNode(py_object_type, may_be_none=True) # Fake rhs to silence "unused variable" warning -fake_rhs_expr = TypedExprNode(unspecified_type) +fake_rhs_expr = TypedExprNode(PyrexTypes.unspecified_type) class ControlBlock(object): @@ -52,7 +52,7 @@ stats = [Assignment(a), NameReference(a), NameReference(c), Assignment(b)] gen = {Entry(a): Assignment(a), Entry(b): Assignment(b)} - bounded = set([Entry(a), Entry(c)]) + bounded = {Entry(a), Entry(c)} """ @@ -160,7 +160,7 @@ (entry.type.is_struct_or_union or entry.type.is_complex or entry.type.is_array or - entry.type.is_cpp_class)): + (entry.type.is_cpp_class and not entry.is_cpp_optional))): # stack allocated structured variable => never uninitialised return True return False @@ -203,7 +203,7 @@ def normalize(self): """Delete unreachable and orphan blocks.""" - queue = set([self.entry_point]) + queue = {self.entry_point} visited = set() while queue: root = queue.pop() @@ -217,7 +217,7 @@ visited.remove(self.entry_point) for block in visited: if block.empty(): - for parent in block.parents: # Re-parent + for parent in block.parents: # Re-parent for child in block.children: parent.add_child(child) block.detach() @@ -341,14 +341,6 @@ return self.entry.type return self.inferred_type - def __getstate__(self): - return (self.lhs, self.rhs, self.entry, self.pos, - self.refs, self.is_arg, self.is_deletion, self.inferred_type) - - def __setstate__(self, state): - (self.lhs, self.rhs, self.entry, self.pos, - self.refs, self.is_arg, self.is_deletion, self.inferred_type) = state - class StaticAssignment(NameAssignment): """Initialised at declaration time, e.g. stack allocation.""" @@ -381,9 +373,9 @@ def infer_type(self): inferred_type = self.rhs.infer_type(self.entry.scope) - if (not inferred_type.is_pyobject and - inferred_type.can_coerce_to_pyobject(self.entry.scope)): - return py_object_type + if (not inferred_type.is_pyobject + and inferred_type.can_coerce_to_pyobject(self.entry.scope)): + return PyrexTypes.py_object_type self.inferred_type = inferred_type return inferred_type @@ -463,7 +455,7 @@ start = min(block.positions) stop = max(block.positions) srcdescr = start[0] - if not srcdescr in self.sources: + if srcdescr not in self.sources: self.sources[srcdescr] = list(srcdescr.get_lines()) lines = self.sources[srcdescr] return '\\n'.join([l.strip() for l in lines[start[1] - 1:stop[1]]]) @@ -598,7 +590,7 @@ if (node.allow_null or entry.from_closure or entry.is_pyclass_attr or entry.type.is_error): pass # Can be uninitialized here - elif node.cf_is_null: + elif node.cf_is_null and not entry.in_closure: if entry.error_on_uninitialized or ( Options.error_on_uninitialized and ( entry.type.is_pyobject or entry.type.is_unspecified)): @@ -612,10 +604,12 @@ "local variable '%s' referenced before assignment" % entry.name) elif warn_maybe_uninitialized: + msg = "local variable '%s' might be referenced before assignment" % entry.name + if entry.in_closure: + msg += " (maybe initialized inside a closure)" messages.warning( node.pos, - "local variable '%s' might be referenced before assignment" - % entry.name) + msg) elif Unknown in node.cf_state: # TODO: better cross-closure analysis to know when inner functions # are being called before a variable is being set, and when @@ -629,7 +623,7 @@ # Unused result for assmt in assignments: if (not assmt.refs and not assmt.entry.is_pyclass_attr - and not assmt.entry.in_closure): + and not assmt.entry.in_closure): if assmt.entry.cf_references and warn_unused_result: if assmt.is_arg: messages.warning(assmt.pos, "Unused argument value '%s'" % @@ -669,7 +663,7 @@ self.assignments = [] def visit_Node(self): - self._visitchildren(self, None) + self._visitchildren(self, None, None) def visit_SingleAssignmentNode(self, node): self.assignments.append((node.lhs, node.rhs)) @@ -682,7 +676,10 @@ class ControlFlowAnalysis(CythonTransform): def visit_ModuleNode(self, node): - self.gv_ctx = GVContext() + dot_output = self.current_directives['control_flow.dot_output'] + self.gv_ctx = GVContext() if dot_output else None + + from .Optimize import ConstantFolding self.constant_folder = ConstantFolding() # Set of NameNode reductions @@ -693,18 +690,15 @@ self.env = node.scope self.stack = [] self.flow = ControlFlow() + self.object_expr = TypedExprNode(PyrexTypes.py_object_type, may_be_none=True) self.visitchildren(node) check_definitions(self.flow, self.current_directives) - dot_output = self.current_directives['control_flow.dot_output'] if dot_output: annotate_defs = self.current_directives['control_flow.dot_annotate_defs'] - fp = open(dot_output, 'wt') - try: + with open(dot_output, 'wt') as fp: self.gv_ctx.render(fp, 'module', annotate_defs=annotate_defs) - finally: - fp.close() return node def visit_FuncDefNode(self, node): @@ -752,7 +746,8 @@ check_definitions(self.flow, self.current_directives) self.flow.blocks.add(self.flow.entry_point) - self.gv_ctx.add(GV(node.local_scope.name, self.flow)) + if self.gv_ctx is not None: + self.gv_ctx.add(GV(node.local_scope.name, self.flow)) self.flow = self.stack.pop() self.env = self.env_stack.pop() @@ -777,19 +772,22 @@ self.flow.nextblock() if not rhs: - rhs = object_expr + rhs = self.object_expr if lhs.is_name: if lhs.entry is not None: entry = lhs.entry else: entry = self.env.lookup(lhs.name) - if entry is None: # TODO: This shouldn't happen... + if entry is None: # TODO: This shouldn't happen... return self.flow.mark_assignment(lhs, rhs, entry) elif lhs.is_sequence_constructor: for i, arg in enumerate(lhs.args): - if not rhs or arg.is_starred: - item_node = None + if arg.is_starred: + # "a, *b = x" assigns a list to "b" + item_node = TypedExprNode(Builtin.list_type, may_be_none=False, pos=arg.pos) + elif rhs is self.object_expr: + item_node = rhs else: item_node = rhs.inferable_item_node(i) self.mark_assignment(arg, item_node) @@ -814,7 +812,7 @@ return node def visit_AssignmentNode(self, node): - raise InternalError("Unhandled assignment node") + raise InternalError("Unhandled assignment node %s" % type(node)) def visit_SingleAssignmentNode(self, node): self._visit(node.rhs) @@ -892,6 +890,12 @@ self.mark_position(node) return node + def visit_SizeofVarNode(self, node): + return node + + def visit_TypeidNode(self, node): + return node + def visit_IfStatNode(self, node): next_block = self.flow.newblock() parent = self.flow.block @@ -918,6 +922,26 @@ self.flow.block = None return node + def visit_AssertStatNode(self, node): + """Essentially an if-condition that wraps a RaiseStatNode. + """ + self.mark_position(node) + next_block = self.flow.newblock() + parent = self.flow.block + # failure case + parent = self.flow.nextblock(parent) + self._visit(node.condition) + self.flow.nextblock() + self._visit(node.exception) + if self.flow.block: + self.flow.block.add_child(next_block) + parent.add_child(next_block) + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node + def visit_WhileStatNode(self, node): condition_block = self.flow.nextblock() next_block = self.flow.newblock() @@ -1015,7 +1039,7 @@ elif isinstance(node, Nodes.AsyncForStatNode): # not entirely correct, but good enough for now self.mark_assignment(node.target, node.item) - else: # Parallel + else: # Parallel self.mark_assignment(node.target) # Body block @@ -1200,7 +1224,6 @@ if self.flow.loops: self.flow.loops[-1].exceptions.append(descr) self.flow.block = body_block - ## XXX: Is it still required body_block.add_child(entry_point) self.flow.nextblock() self._visit(node.body) @@ -1235,11 +1258,18 @@ self.mark_position(node) self.visitchildren(node) - for exception in self.flow.exceptions[::-1]: - if exception.finally_enter: - self.flow.block.add_child(exception.finally_enter) - if exception.finally_exit: - exception.finally_exit.add_child(self.flow.exit_point) + outer_exception_handlers = iter(self.flow.exceptions[::-1]) + for handler in outer_exception_handlers: + if handler.finally_enter: + self.flow.block.add_child(handler.finally_enter) + if handler.finally_exit: + # 'return' goes to function exit, or to the next outer 'finally' clause + exit_point = self.flow.exit_point + for next_handler in outer_exception_handlers: + if next_handler.finally_enter: + exit_point = next_handler.finally_enter + break + handler.finally_exit.add_child(exit_point) break else: if self.flow.block: @@ -1304,10 +1334,12 @@ self.visitchildren(node, ('dict', 'metaclass', 'mkw', 'bases', 'class_result')) self.flow.mark_assignment(node.target, node.classobj, - self.env.lookup(node.name)) + self.env.lookup(node.target.name)) self.env_stack.append(self.env) self.env = node.scope self.flow.nextblock() + if node.doc_node: + self.flow.mark_assignment(node.doc_node, fake_rhs_expr, node.doc_node.entry) self.visitchildren(node, ('body',)) self.flow.nextblock() self.env = self.env_stack.pop() diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/FusedNode.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/FusedNode.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/FusedNode.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/FusedNode.py 2022-03-24 10:16:46.000000000 +0000 @@ -6,6 +6,8 @@ ParseTreeTransforms, StringEncoding, Errors) from .ExprNodes import CloneNode, ProxyNode, TupleNode from .Nodes import FuncDefNode, CFuncDefNode, StatListNode, DefNode +from ..Utils import OrderedSet +from .Errors import error, CannotSpecialize class FusedCFuncDefNode(StatListNode): @@ -126,9 +128,6 @@ # len(permutations)) # import pprint; pprint.pprint([d for cname, d in permutations]) - if self.node.entry in env.cfunc_entries: - env.cfunc_entries.remove(self.node.entry) - # Prevent copying of the python function self.orig_py_func = orig_py_func = self.node.py_func self.node.py_func = None @@ -138,12 +137,33 @@ fused_types = self.node.type.get_fused_types() self.fused_compound_types = fused_types + new_cfunc_entries = [] for cname, fused_to_specific in permutations: copied_node = copy.deepcopy(self.node) - # Make the types in our CFuncType specific - type = copied_node.type.specialize(fused_to_specific) + # Make the types in our CFuncType specific. + try: + type = copied_node.type.specialize(fused_to_specific) + except CannotSpecialize: + # unlike for the argument types, specializing the return type can fail + error(copied_node.pos, "Return type is a fused type that cannot " + "be determined from the function arguments") + self.py_func = None # this is just to let the compiler exit gracefully + return entry = copied_node.entry + type.specialize_entry(entry, cname) + + # Reuse existing Entries (e.g. from .pxd files). + for i, orig_entry in enumerate(env.cfunc_entries): + if entry.cname == orig_entry.cname and type.same_as_resolved_type(orig_entry.type): + copied_node.entry = env.cfunc_entries[i] + if not copied_node.entry.func_cname: + copied_node.entry.func_cname = entry.func_cname + entry = copied_node.entry + type = entry.type + break + else: + new_cfunc_entries.append(entry) copied_node.type = type entry.type, type.entry = type, entry @@ -164,9 +184,6 @@ self._specialize_function_args(copied_node.cfunc_declarator.args, fused_to_specific) - type.specialize_entry(entry, cname) - env.cfunc_entries.append(entry) - # If a cpdef, declare all specialized cpdefs (this # also calls analyse_declarations) copied_node.declare_cpdef_wrapper(env) @@ -180,6 +197,14 @@ if not self.replace_fused_typechecks(copied_node): break + # replace old entry with new entries + try: + cindex = env.cfunc_entries.index(self.node.entry) + except ValueError: + env.cfunc_entries.extend(new_cfunc_entries) + else: + env.cfunc_entries[cindex:cindex+1] = new_cfunc_entries + if orig_py_func: self.py_func = self.make_fused_cpdef(orig_py_func, env, is_def=False) @@ -203,12 +228,16 @@ arg.type = arg.type.specialize(fused_to_specific) if arg.type.is_memoryviewslice: arg.type.validate_memslice_dtype(arg.pos) + if arg.annotation: + # TODO might be nice if annotations were specialized instead? + # (Or might be hard to do reliably) + arg.annotation.untyped = True def create_new_local_scope(self, node, env, f2s): """ Create a new local scope for the copied node and append it to self.nodes. A new local scope is needed because the arguments with the - fused types are aready in the local scope, and we need the specialized + fused types are already in the local scope, and we need the specialized entries created after analyse_declarations on each specialized version of the (CFunc)DefNode. f2s is a dict mapping each fused type to its specialized version @@ -247,19 +276,19 @@ Returns whether an error was issued and whether we should stop in in order to prevent a flood of errors. """ - num_errors = Errors.num_errors + num_errors = Errors.get_errors_count() transform = ParseTreeTransforms.ReplaceFusedTypeChecks( copied_node.local_scope) transform(copied_node) - if Errors.num_errors > num_errors: + if Errors.get_errors_count() > num_errors: return False return True def _fused_instance_checks(self, normal_types, pyx_code, env): """ - Genereate Cython code for instance checks, matching an object to + Generate Cython code for instance checks, matching an object to specialized types. """ for specialized_type in normal_types: @@ -314,19 +343,22 @@ match = "dest_sig[{{dest_sig_idx}}] = '{{specialized_type_name}}'" no_match = "dest_sig[{{dest_sig_idx}}] = None" - def _buffer_check_numpy_dtype(self, pyx_code, specialized_buffer_types): + def _buffer_check_numpy_dtype(self, pyx_code, specialized_buffer_types, pythran_types): """ Match a numpy dtype object to the individual specializations. """ self._buffer_check_numpy_dtype_setup_cases(pyx_code) - for specialized_type in specialized_buffer_types: + for specialized_type in pythran_types+specialized_buffer_types: + final_type = specialized_type + if specialized_type.is_pythran_expr: + specialized_type = specialized_type.org_buffer dtype = specialized_type.dtype pyx_code.context.update( itemsize_match=self._sizeof_dtype(dtype) + " == itemsize", signed_match="not (%s_is_signed ^ dtype_signed)" % self._dtype_name(dtype), dtype=dtype, - specialized_type_name=specialized_type.specialization_string) + specialized_type_name=final_type.specialization_string) dtypes = [ (dtype.is_int, pyx_code.dtype_int), @@ -341,8 +373,11 @@ if dtype.is_int: cond += ' and {{signed_match}}' + if final_type.is_pythran_expr: + cond += ' and arg_is_pythran_compatible' + if codewriter.indenter("if %s:" % cond): - # codewriter.putln("print 'buffer match found based on numpy dtype'") + #codewriter.putln("print 'buffer match found based on numpy dtype'") codewriter.putln(self.match) codewriter.putln("break") codewriter.dedent() @@ -367,7 +402,7 @@ coerce_from_py_func=memslice_type.from_py_function, dtype=dtype) decl_code.putln( - "{{memviewslice_cname}} {{coerce_from_py_func}}(object)") + "{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)") pyx_code.context.update( specialized_type_name=specialized_type.specialization_string, @@ -377,9 +412,9 @@ u""" # try {{dtype}} if itemsize == -1 or itemsize == {{sizeof_dtype}}: - memslice = {{coerce_from_py_func}}(arg) + memslice = {{coerce_from_py_func}}(arg, 0) if memslice.memview: - __PYX_XDEC_MEMVIEW(&memslice, 1) + __PYX_XCLEAR_MEMVIEW(&memslice, 1) # print 'found a match for the buffer through format parsing' %s break @@ -387,7 +422,7 @@ __pyx_PyErr_Clear() """ % self.match) - def _buffer_checks(self, buffer_types, pyx_code, decl_code, env): + def _buffer_checks(self, buffer_types, pythran_types, pyx_code, decl_code, env): """ Generate Cython code to match objects to buffer specializations. First try to get a numpy dtype object and match it against the individual @@ -398,9 +433,11 @@ # The first thing to find a match in this loop breaks out of the loop pyx_code.put_chunk( u""" + """ + (u"arg_is_pythran_compatible = False" if pythran_types else u"") + u""" if ndarray is not None: if isinstance(arg, ndarray): dtype = arg.dtype + """ + (u"arg_is_pythran_compatible = True" if pythran_types else u"") + u""" elif __pyx_memoryview_check(arg): arg_base = arg.base if isinstance(arg_base, ndarray): @@ -417,15 +454,36 @@ dtype_signed = kind == 'i' """) pyx_code.indent(2) + if pythran_types: + pyx_code.put_chunk( + u""" + # Pythran only supports the endianness of the current compiler + byteorder = dtype.byteorder + if byteorder == "<" and not __Pyx_Is_Little_Endian(): + arg_is_pythran_compatible = False + elif byteorder == ">" and __Pyx_Is_Little_Endian(): + arg_is_pythran_compatible = False + if arg_is_pythran_compatible: + cur_stride = itemsize + shape = arg.shape + strides = arg.strides + for i in range(arg.ndim-1, -1, -1): + if (strides[i]) != cur_stride: + arg_is_pythran_compatible = False + break + cur_stride *= shape[i] + else: + arg_is_pythran_compatible = not (arg.flags.f_contiguous and (arg.ndim) > 1) + """) pyx_code.named_insertion_point("numpy_dtype_checks") - self._buffer_check_numpy_dtype(pyx_code, buffer_types) + self._buffer_check_numpy_dtype(pyx_code, buffer_types, pythran_types) pyx_code.dedent(2) for specialized_type in buffer_types: self._buffer_parse_format_string_check( pyx_code, decl_code, specialized_type, env) - def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types): + def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types, pythran_types): """ If we have any buffer specializations, write out some variable declarations and imports. @@ -435,7 +493,7 @@ ctypedef struct {{memviewslice_cname}}: void *memview - void __PYX_XDEC_MEMVIEW({{memviewslice_cname}} *, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_cname}} *, int have_gil) bint __pyx_memoryview_check(object) """) @@ -449,35 +507,39 @@ itemsize = -1 """) + if pythran_types: + pyx_code.local_variable_declarations.put_chunk(u""" + cdef bint arg_is_pythran_compatible + cdef Py_ssize_t cur_stride + """) + pyx_code.imports.put_chunk( u""" cdef type ndarray - try: - import numpy - ndarray = numpy.ndarray - except (ImportError, AttributeError, TypeError): - ndarray = None + ndarray = __Pyx_ImportNumPyArrayTypeIfAvailable() """) + seen_typedefs = set() seen_int_dtypes = set() for buffer_type in all_buffer_types: dtype = buffer_type.dtype + dtype_name = self._dtype_name(dtype) if dtype.is_typedef: - #decl_code.putln("ctypedef %s %s" % (dtype.resolve(), - # self._dtype_name(dtype))) - decl_code.putln('ctypedef %s %s "%s"' % (dtype.resolve(), - self._dtype_name(dtype), - dtype.empty_declaration_code())) + if dtype_name not in seen_typedefs: + seen_typedefs.add(dtype_name) + decl_code.putln( + 'ctypedef %s %s "%s"' % (dtype.resolve(), dtype_name, + dtype.empty_declaration_code())) if buffer_type.dtype.is_int: if str(dtype) not in seen_int_dtypes: seen_int_dtypes.add(str(dtype)) - pyx_code.context.update(dtype_name=self._dtype_name(dtype), + pyx_code.context.update(dtype_name=dtype_name, dtype_type=self._dtype_type(dtype)) pyx_code.local_variable_declarations.put_chunk( u""" cdef bint {{dtype_name}}_is_signed - {{dtype_name}}_is_signed = <{{dtype_type}}> -1 < 0 + {{dtype_name}}_is_signed = not (<{{dtype_type}}> -1 > 0) """) def _split_fused_types(self, arg): @@ -490,7 +552,7 @@ specialized_types.sort() seen_py_type_names = set() - normal_types, buffer_types = [], [] + normal_types, buffer_types, pythran_types = [], [], [] has_object_fallback = False for specialized_type in specialized_types: py_type_name = specialized_type.py_type_name() @@ -502,10 +564,12 @@ has_object_fallback = True else: normal_types.append(specialized_type) + elif specialized_type.is_pythran_expr: + pythran_types.append(specialized_type) elif specialized_type.is_buffer or specialized_type.is_memoryviewslice: buffer_types.append(specialized_type) - return normal_types, buffer_types, has_object_fallback + return normal_types, buffer_types, pythran_types, has_object_fallback def _unpack_argument(self, pyx_code): pyx_code.put_chunk( @@ -513,16 +577,41 @@ # PROCESSING ARGUMENT {{arg_tuple_idx}} if {{arg_tuple_idx}} < len(args): arg = (args)[{{arg_tuple_idx}}] - elif '{{arg.name}}' in kwargs: + elif kwargs is not None and '{{arg.name}}' in kwargs: arg = (kwargs)['{{arg.name}}'] else: {{if arg.default}} arg = (defaults)[{{default_idx}}] {{else}} - raise TypeError("Expected at least %d arguments" % len(args)) + {{if arg_tuple_idx < min_positional_args}} + raise TypeError("Expected at least %d argument%s, got %d" % ( + {{min_positional_args}}, {{'"s"' if min_positional_args != 1 else '""'}}, len(args))) + {{else}} + raise TypeError("Missing keyword-only argument: '%s'" % "{{arg.default}}") + {{endif}} {{endif}} """) + def _fused_signature_index(self, pyx_code): + """ + Generate Cython code for constructing a persistent nested dictionary index of + fused type specialization signatures. + """ + pyx_code.put_chunk( + u""" + if not _fused_sigindex: + for sig in signatures: + sigindex_node = _fused_sigindex + *sig_series, last_type = sig.strip('()').split('|') + for sig_type in sig_series: + if sig_type not in sigindex_node: + sigindex_node[sig_type] = sigindex_node = {} + else: + sigindex_node = sigindex_node[sig_type] + sigindex_node[last_type] = sig + """ + ) + def make_fused_cpdef(self, orig_py_func, env, is_def): """ This creates the function that is indexable from Python and does @@ -539,6 +628,10 @@ 'memviewslice_cname': MemoryView.memviewslice_cname, 'func_args': self.node.args, 'n_fused': len(fused_types), + 'min_positional_args': + self.node.num_required_args - self.node.num_required_kw_args + if is_def else + sum(1 for arg in self.node.args if arg.default is None), 'name': orig_py_func.entry.name, } @@ -548,35 +641,39 @@ u""" cdef extern from *: void __pyx_PyErr_Clear "PyErr_Clear" () + type __Pyx_ImportNumPyArrayTypeIfAvailable() + int __Pyx_Is_Little_Endian() """) decl_code.indent() pyx_code.put_chunk( u""" - from __future__ import absolute_import # for later numpy import - """) - pyx_code.put_chunk( - u""" - def __pyx_fused_cpdef(signatures, args, kwargs, defaults): + def __pyx_fused_cpdef(signatures, args, kwargs, defaults, _fused_sigindex={}): # FIXME: use a typed signature - currently fails badly because # default arguments inherit the types we specify here! + cdef list search_list + + cdef dict sn, sigindex_node + dest_sig = [None] * {{n_fused}} - if kwargs is None: - kwargs = {} + if kwargs is not None and not kwargs: + kwargs = None cdef Py_ssize_t i # instance check body """) - pyx_code.indent() # indent following code to function body + + pyx_code.indent() # indent following code to function body pyx_code.named_insertion_point("imports") + pyx_code.named_insertion_point("func_defs") pyx_code.named_insertion_point("local_variable_declarations") fused_index = 0 default_idx = 0 - all_buffer_types = set() + all_buffer_types = OrderedSet() seen_fused_types = set() for i, arg in enumerate(self.node.args): if arg.type.is_fused: @@ -596,15 +693,16 @@ default_idx=default_idx, ) - normal_types, buffer_types, has_object_fallback = self._split_fused_types(arg) + normal_types, buffer_types, pythran_types, has_object_fallback = self._split_fused_types(arg) self._unpack_argument(pyx_code) # 'unrolled' loop, first match breaks out of it if pyx_code.indenter("while 1:"): if normal_types: self._fused_instance_checks(normal_types, pyx_code, env) - if buffer_types: - self._buffer_checks(buffer_types, pyx_code, decl_code, env) + if buffer_types or pythran_types: + env.use_utility_code(Code.UtilityCode.load_cached("IsLittleEndian", "ModuleSetupCode.c")) + self._buffer_checks(buffer_types, pythran_types, pyx_code, decl_code, env) if has_object_fallback: pyx_code.context.update(specialized_type_name='object') pyx_code.putln(self.match) @@ -615,29 +713,46 @@ fused_index += 1 all_buffer_types.update(buffer_types) + all_buffer_types.update(ty.org_buffer for ty in pythran_types) if arg.default: default_idx += 1 if all_buffer_types: - self._buffer_declarations(pyx_code, decl_code, all_buffer_types) + self._buffer_declarations(pyx_code, decl_code, all_buffer_types, pythran_types) env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c")) + env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c")) + + self._fused_signature_index(pyx_code) pyx_code.put_chunk( u""" - candidates = [] - for sig in signatures: - match_found = False - for src_type, dst_type in zip(sig.strip('()').split('|'), dest_sig): - if dst_type is not None: - if src_type == dst_type: - match_found = True - else: - match_found = False - break + sigindex_matches = [] + sigindex_candidates = [_fused_sigindex] - if match_found: - candidates.append(sig) + for dst_type in dest_sig: + found_matches = [] + found_candidates = [] + # Make two seperate lists: One for signature sub-trees + # with at least one definite match, and another for + # signature sub-trees with only ambiguous matches + # (where `dest_sig[i] is None`). + if dst_type is None: + for sn in sigindex_matches: + found_matches.extend(sn.values()) + for sn in sigindex_candidates: + found_candidates.extend(sn.values()) + else: + for search_list in (sigindex_matches, sigindex_candidates): + for sn in search_list: + if dst_type in sn: + found_matches.append(sn[dst_type]) + sigindex_matches = found_matches + sigindex_candidates = found_candidates + if not (found_matches or found_candidates): + break + + candidates = sigindex_matches if not candidates: raise TypeError("No matching signature found") @@ -726,16 +841,18 @@ for arg in self.node.args: if arg.default: arg.default = arg.default.analyse_expressions(env) - defaults.append(ProxyNode(arg.default)) + # coerce the argument to temp since CloneNode really requires a temp + defaults.append(ProxyNode(arg.default.coerce_to_temp(env))) else: defaults.append(None) for i, stat in enumerate(self.stats): stat = self.stats[i] = stat.analyse_expressions(env) - if isinstance(stat, FuncDefNode): + if isinstance(stat, FuncDefNode) and stat is not self.py_func: + # the dispatcher specifically doesn't want its defaults overriding for arg, default in zip(stat.args, defaults): if default is not None: - arg.default = CloneNode(default).coerce_to(arg.type, env) + arg.default = CloneNode(default).analyse_expressions(env).coerce_to(arg.type, env) if self.py_func: args = [CloneNode(default) for default in defaults if default] @@ -763,6 +880,10 @@ else: nodes = self.nodes + # For the moment, fused functions do not support METH_FASTCALL + for node in nodes: + node.entry.signature.use_fastcall = False + signatures = [StringEncoding.EncodedString(node.specialized_signature_string) for node in nodes] keys = [ExprNodes.StringNode(node.pos, value=sig) @@ -811,18 +932,24 @@ "((__pyx_FusedFunctionObject *) %s)->__signatures__ = %s;" % (self.resulting_fused_function.result(), self.__signatures__.result())) - code.put_giveref(self.__signatures__.result()) + self.__signatures__.generate_giveref(code) + self.__signatures__.generate_post_assignment_code(code) + self.__signatures__.free_temps(code) self.fused_func_assignment.generate_execution_code(code) # Dispose of results self.resulting_fused_function.generate_disposal_code(code) + self.resulting_fused_function.free_temps(code) self.defaults_tuple.generate_disposal_code(code) + self.defaults_tuple.free_temps(code) self.code_object.generate_disposal_code(code) + self.code_object.free_temps(code) for default in self.defaults: if default is not None: default.generate_disposal_code(code) + default.free_temps(code) def annotate(self, code): for stat in self.stats: diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Future.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Future.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Future.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Future.py 2022-03-24 10:16:46.000000000 +0000 @@ -4,12 +4,13 @@ return getattr(__future__, name, object()) unicode_literals = _get_feature("unicode_literals") -with_statement = _get_feature("with_statement") +with_statement = _get_feature("with_statement") # dummy division = _get_feature("division") print_function = _get_feature("print_function") absolute_import = _get_feature("absolute_import") nested_scopes = _get_feature("nested_scopes") # dummy generators = _get_feature("generators") # dummy generator_stop = _get_feature("generator_stop") +annotations = _get_feature("annotations") del _get_feature diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Interpreter.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Interpreter.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Interpreter.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Interpreter.py 2022-03-24 10:16:46.000000000 +0000 @@ -47,8 +47,8 @@ raise CompileError(node.pos, "Type not allowed here.") else: if (sys.version_info[0] >=3 and - isinstance(node, StringNode) and - node.unicode_value is not None): + isinstance(node, StringNode) and + node.unicode_value is not None): return (node.unicode_value, node.pos) return (node.compile_time_value(empty_scope), node.pos) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Lexicon.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Lexicon.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Lexicon.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Lexicon.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,9 +1,10 @@ +# -*- coding: utf-8 -*- # cython: language_level=3, py2_import=True # # Cython Scanner - Lexical Definitions # -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals raw_prefixes = "rR" bytes_prefixes = "bB" @@ -16,28 +17,43 @@ def make_lexicon(): from ..Plex import \ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \ - TEXT, IGNORE, State, Lexicon - from .Scanning import Method + TEXT, IGNORE, Method, State, Lexicon, Range - letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") + nonzero_digit = Any("123456789") digit = Any("0123456789") bindigit = Any("01") octdigit = Any("01234567") hexdigit = Any("0123456789ABCDEFabcdef") indentation = Bol + Rep(Any(" \t")) + # The list of valid unicode identifier characters are pretty slow to generate at runtime, + # and require Python3, so are just included directly here + # (via the generated code block at the bottom of the file) + unicode_start_character = (Any(unicode_start_ch_any) | Range(unicode_start_ch_range)) + unicode_continuation_character = ( + unicode_start_character | + Any(unicode_continuation_ch_any) | Range(unicode_continuation_ch_range)) + def underscore_digits(d): return Rep1(d) + Rep(Str("_") + Rep1(d)) + def prefixed_digits(prefix, digits): + return prefix + Opt(Str("_")) + underscore_digits(digits) + decimal = underscore_digits(digit) dot = Str(".") exponent = Any("Ee") + Opt(Any("+-")) + decimal decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal) - name = letter + Rep(letter | digit) - intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) | - (Any("Oo") + underscore_digits(octdigit)) | - (Any("Bb") + underscore_digits(bindigit)) )) + #name = letter + Rep(letter | digit) + name = unicode_start_character + Rep(unicode_continuation_character) + intconst = (prefixed_digits(nonzero_digit, digit) | # decimal literals with underscores must not start with '0' + (Str("0") + (prefixed_digits(Any("Xx"), hexdigit) | + prefixed_digits(Any("Oo"), octdigit) | + prefixed_digits(Any("Bb"), bindigit) )) | + underscore_digits(Str('0')) # 0_0_0_0... is allowed as a decimal literal + | Rep1(digit) # FIXME: remove these Py2 style decimal/octal literals (PY_VERSION_HEX < 3) + ) intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu"))) intliteral = intconst + intsuffix fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) @@ -61,7 +77,7 @@ punct = Any(":,;+-*/|&<>=.%`~^?!@") diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", - "<<=", ">>=", "**=", "//=", "->", "@=") + "<<=", ">>=", "**=", "//=", "->", "@=", "&&", "||", ':=') spaces = Rep1(Any(" \t\f")) escaped_newline = Str("\\\n") lineterm = Eol + Opt(Str("\n")) @@ -69,7 +85,7 @@ comment = Str("#") + Rep(AnyBut("\n")) return Lexicon([ - (name, IDENT), + (name, Method('normalize_ident')), (intliteral, Method('strip_underscores', symbol='INT')), (fltconst, Method('strip_underscores', symbol='FLOAT')), (imagconst, Method('strip_underscores', symbol='IMAG')), @@ -136,3 +152,50 @@ #debug_file = scanner_dump_file ) + +# BEGIN GENERATED CODE +# generated with: +# cpython 3.10.0a0 (heads/master:2b0e654f91, May 29 2020, 16:17:52) + +unicode_start_ch_any = ( + u"_ªµºˬˮͿΆΌՙەۿܐޱߺࠚࠤࠨऽॐলঽৎৼਫ਼ઽૐૹଽୱஃஜௐఽಀಽೞഽൎලาຄລາຽໆༀဿၡႎჇჍቘዀៗៜᢪᪧᳺὙ" + u"ὛὝιⁱⁿℂℇℕℤΩℨⅎⴧⴭⵯꣻꧏꩺꪱꫀꫂיִמּﹱﹳﹷﹹﹻﹽ𐠈𐠼𐨀𐼧𑅄𑅇𑅶𑇚𑇜𑊈𑌽𑍐𑓇𑙄𑚸𑤉𑤿𑥁𑧡𑧣𑨀𑨺𑩐𑪝𑱀𑵆𑶘𑾰𖽐𖿣𝒢" + u"𝒻𝕆𞅎𞥋𞸤𞸧𞸹𞸻𞹂𞹇𞹉𞹋𞹔𞹗𞹙𞹛𞹝𞹟𞹤𞹾" +) +unicode_start_ch_range = ( + u"AZazÀÖØöøˁˆˑˠˤͰʹͶͷͻͽΈΊΎΡΣϵϷҁҊԯԱՖՠֈאתׯײؠيٮٯٱۓۥۦۮۯۺۼܒܯݍޥߊߪߴߵࠀࠕ" + u"ࡀࡘࡠࡪࢠࢴࢶࣇऄहक़ॡॱঀঅঌএঐওনপরশহড়ঢ়য়ৡৰৱਅਊਏਐਓਨਪਰਲਲ਼ਵਸ਼ਸਹਖ਼ੜੲੴઅઍએઑઓનપરલળવહ" + u"ૠૡଅଌଏଐଓନପରଲଳଵହଡ଼ଢ଼ୟୡஅஊஎஐஒகஙசஞடணதநபமஹఅఌఎఐఒనపహౘౚౠౡಅಌಎಐಒನಪಳವಹೠೡೱೲ" + u"ഄഌഎഐഒഺൔൖൟൡൺൿඅඖකනඳරවෆกะเๆກຂຆຊຌຣວະເໄໜໟཀཇཉཬྈྌကဪၐၕၚၝၥၦၮၰၵႁႠჅაჺჼቈ" + u"ቊቍቐቖቚቝበኈኊኍነኰኲኵኸኾዂዅወዖዘጐጒጕጘፚᎀᎏᎠᏵᏸᏽᐁᙬᙯᙿᚁᚚᚠᛪᛮᛸᜀᜌᜎᜑᜠᜱᝀᝑᝠᝬᝮᝰកឳᠠᡸᢀᢨ" + u"ᢰᣵᤀᤞᥐᥭᥰᥴᦀᦫᦰᧉᨀᨖᨠᩔᬅᬳᭅᭋᮃᮠᮮᮯᮺᯥᰀᰣᱍᱏᱚᱽᲀᲈᲐᲺᲽᲿᳩᳬᳮᳳᳵᳶᴀᶿḀἕἘἝἠὅὈὍὐὗὟώᾀᾴ" + u"ᾶᾼῂῄῆῌῐΐῖΊῠῬῲῴῶῼₐₜℊℓ℘ℝKℹℼℿⅅⅉⅠↈⰀⰮⰰⱞⱠⳤⳫⳮⳲⳳⴀⴥⴰⵧⶀⶖⶠⶦⶨⶮⶰⶶⶸⶾⷀⷆⷈⷎⷐⷖ" + u"ⷘⷞ々〇〡〩〱〵〸〼ぁゖゝゟァヺーヿㄅㄯㄱㆎㆠㆿㇰㇿ㐀䶿一鿼ꀀꒌꓐꓽꔀꘌꘐꘟꘪꘫꙀꙮꙿꚝꚠꛯꜗꜟꜢꞈꞋꞿꟂꟊꟵꠁꠃꠅꠇꠊ" + u"ꠌꠢꡀꡳꢂꢳꣲꣷꣽꣾꤊꤥꤰꥆꥠꥼꦄꦲꧠꧤꧦꧯꧺꧾꨀꨨꩀꩂꩄꩋꩠꩶꩾꪯꪵꪶꪹꪽꫛꫝꫠꫪꫲꫴꬁꬆꬉꬎꬑꬖꬠꬦꬨꬮꬰꭚꭜꭩꭰꯢ" + u"가힣ힰퟆퟋퟻ豈舘並龎ffstﬓﬗײַﬨשׁזּטּלּנּסּףּפּצּﮱﯓﱝﱤﴽﵐﶏﶒﷇﷰﷹﹿﻼＡＺａｚｦﾝﾠﾾￂￇￊￏￒￗￚￜ𐀀𐀋𐀍𐀦𐀨𐀺" + u"𐀼𐀽𐀿𐁍𐁐𐁝𐂀𐃺𐅀𐅴𐊀𐊜𐊠𐋐𐌀𐌟𐌭𐍊𐍐𐍵𐎀𐎝𐎠𐏃𐏈𐏏𐏑𐏕𐐀𐒝𐒰𐓓𐓘𐓻𐔀𐔧𐔰𐕣𐘀𐜶𐝀𐝕𐝠𐝧𐠀𐠅𐠊𐠵𐠷𐠸𐠿𐡕𐡠𐡶𐢀𐢞𐣠𐣲𐣴𐣵" + u"𐤀𐤕𐤠𐤹𐦀𐦷𐦾𐦿𐨐𐨓𐨕𐨗𐨙𐨵𐩠𐩼𐪀𐪜𐫀𐫇𐫉𐫤𐬀𐬵𐭀𐭕𐭠𐭲𐮀𐮑𐰀𐱈𐲀𐲲𐳀𐳲𐴀𐴣𐺀𐺩𐺰𐺱𐼀𐼜𐼰𐽅𐾰𐿄𐿠𐿶𑀃𑀷𑂃𑂯𑃐𑃨𑄃𑄦𑅐𑅲" + u"𑆃𑆲𑇁𑇄𑈀𑈑𑈓𑈫𑊀𑊆𑊊𑊍𑊏𑊝𑊟𑊨𑊰𑋞𑌅𑌌𑌏𑌐𑌓𑌨𑌪𑌰𑌲𑌳𑌵𑌹𑍝𑍡𑐀𑐴𑑇𑑊𑑟𑑡𑒀𑒯𑓄𑓅𑖀𑖮𑗘𑗛𑘀𑘯𑚀𑚪𑜀𑜚𑠀𑠫𑢠𑣟𑣿𑤆𑤌𑤓" + u"𑤕𑤖𑤘𑤯𑦠𑦧𑦪𑧐𑨋𑨲𑩜𑪉𑫀𑫸𑰀𑰈𑰊𑰮𑱲𑲏𑴀𑴆𑴈𑴉𑴋𑴰𑵠𑵥𑵧𑵨𑵪𑶉𑻠𑻲𒀀𒎙𒐀𒑮𒒀𒕃𓀀𓐮𔐀𔙆𖠀𖨸𖩀𖩞𖫐𖫭𖬀𖬯𖭀𖭃𖭣𖭷𖭽𖮏𖹀𖹿" + u"𖼀𖽊𖾓𖾟𖿠𖿡𗀀𘟷𘠀𘳕𘴀𘴈𛀀𛄞𛅐𛅒𛅤𛅧𛅰𛋻𛰀𛱪𛱰𛱼𛲀𛲈𛲐𛲙𝐀𝑔𝑖𝒜𝒞𝒟𝒥𝒦𝒩𝒬𝒮𝒹𝒽𝓃𝓅𝔅𝔇𝔊𝔍𝔔𝔖𝔜𝔞𝔹𝔻𝔾𝕀𝕄𝕊𝕐𝕒𝚥" + u"𝚨𝛀𝛂𝛚𝛜𝛺𝛼𝜔𝜖𝜴𝜶𝝎𝝐𝝮𝝰𝞈𝞊𝞨𝞪𝟂𝟄𝟋𞄀𞄬𞄷𞄽𞋀𞋫𞠀𞣄𞤀𞥃𞸀𞸃𞸅𞸟𞸡𞸢𞸩𞸲𞸴𞸷𞹍𞹏𞹑𞹒𞹡𞹢𞹧𞹪𞹬𞹲𞹴𞹷𞹹𞹼𞺀𞺉𞺋𞺛" + u"𞺡𞺣𞺥𞺩𞺫𞺻𠀀𪛝𪜀𫜴𫝀𫠝𫠠𬺡𬺰𮯠丽𪘀" +) +unicode_continuation_ch_any = ( + u"··়ׇֿٰܑ߽ৗ਼৾ੑੵ઼଼ஂௗ಼ൗ්ූัັ༹༵༷࿆᳭ᢩ៝᳴⁔⵿⃡꙯ꠂ꠆ꠋ꠬ꧥꩃﬞꪰ꫁＿𑅳𐨿𐇽𐋠𑈾𑍗𑑞𑥀𑧤𑩇𑴺𑵇𖽏𖿤𝩵" + u"𝪄" +) +unicode_continuation_ch_range = ( + u"09ֽׁׂًؚ֑ׅ̀ͯ҃҇ׄؐ٩۪ۭۖۜ۟ۤۧۨ۰۹ܰ݊ަް߀߉࡙࡛࣓ࣣ߫߳ࠖ࠙ࠛࠣࠥࠧࠩ࠭࣡ःऺ़ाॏ॑ॗॢॣ०९ঁঃ" + u"াৄেৈো্ৢৣ০৯ਁਃਾੂੇੈੋ੍੦ੱઁઃાૅેૉો્ૢૣ૦૯ૺ૿ଁଃାୄେୈୋ୍୕ୗୢୣ୦୯ாூெைொ்௦௯ఀఄాౄ" + u"ెైొ్ౕౖౢౣ౦౯ಁಃಾೄೆೈೊ್ೕೖೢೣ೦೯ഀഃ഻഼ാൄെൈൊ്ൢൣ൦൯ඁඃාුෘෟ෦෯ෲෳำฺ็๎๐๙ຳຼ່ໍ໐໙" + u"༘༙༠༩༾༿྄ཱ྆྇ྍྗྙྼါှ၀၉ၖၙၞၠၢၤၧၭၱၴႂႍႏႝ፝፟፩፱ᜒ᜔ᜲ᜴ᝒᝓᝲᝳ឴៓០៩᠋᠍᠐᠙ᤠᤫᤰ᤻᥆᥏᧐᧚" + u"ᨗᨛᩕᩞ᩠᩿᩼᪉᪐᪙᪽ᪿᫀ᪰ᬀᬄ᬴᭄᭐᭙᭫᭳ᮀᮂᮡᮭ᮰᮹᯦᯳ᰤ᰷᱀᱉᱐᱙᳔᳨᳐᳒᳷᷹᷿᳹᷀᷻‿⁀⃥゙゚〪〯⃐⃜⃰⳯⳱ⷠⷿ" + u"꘠꘩ꙴ꙽ꚞꚟ꛰꛱ꠣꠧꢀꢁꢴꣅ꣐꣙꣠꣱ꣿ꤉ꤦ꤭ꥇ꥓ꦀꦃ꦳꧀꧐꧙꧰꧹ꨩꨶꩌꩍ꩐꩙ꩻꩽꪴꪲꪷꪸꪾ꪿ꫫꫯꫵ꫶ꯣꯪ꯬꯭꯰꯹︀️︠︯" + u"︳︴﹍﹏０９ﾞﾟ𐍶𐍺𐒠𐒩𐨁𐨃𐨅𐨆𐨌𐨺𐫦𐨏𐨸𐫥𐴤𐴧𐴰𐴹𐽆𐽐𐺫𐺬𑀀𑀂𑀸𑁆𑁦𑁯𑁿𑂂𑂰𑂺𑃰𑃹𑄀𑄂𑄧𑄴𑄶𑄿𑅅𑅆𑆀𑆂𑆳𑇀𑇉𑇌𑇎𑇙𑈬𑈷" + u"𑋟𑋪𑋰𑋹𑌀𑌃𑌻𑌼𑌾𑍄𑍇𑍈𑍋𑍍𑍢𑍣𑍦𑍬𑍰𑍴𑐵𑑆𑑐𑑙𑒰𑓃𑓐𑓙𑖯𑖵𑖸𑗀𑗜𑗝𑘰𑙀𑙐𑙙𑚫𑚷𑛀𑛉𑜝𑜫𑜰𑜹𑠬𑠺𑣠𑣩𑤰𑤵𑤷𑤸𑤻𑤾𑥂𑥃𑥐𑥙" + u"𑧑𑧗𑧚𑧠𑨁𑨊𑨳𑨹𑨻𑨾𑩑𑩛𑪊𑪙𑰯𑰶𑰸𑰿𑱐𑱙𑲒𑲧𑲩𑲶𑴱𑴶𑴼𑴽𑴿𑵅𑵐𑵙𑶊𑶎𑶐𑶑𑶓𑶗𑶠𑶩𑻳𑻶𖩠𖩩𖫰𖫴𖬰𖬶𖭐𖭙𖽑𖾇𖾏𖾒𖿰𖿱𛲝𛲞𝅩𝅥" + u"𝅲𝅻𝆂𝆋𝅭𝆅𝆪𝆭𝉂𝉄𝟎𝟿𝨀𝨶𝨻𝩬𝪛𝪟𝪡𝪯𞀀𞀆𞀈𞀘𞀛𞀡𞀣𞀤𞀦𞀪𞄰𞄶𞅀𞅉𞋬𞋹𞥊𞣐𞣖𞥄𞥐𞥙🯰🯹" +) + +# END GENERATED CODE diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Compiler/Main.py cython-0.20.1+1~202203241016-9537/Cython/Compiler/Main.py --- cython-0.20.1+1~201611251650-6686/Cython/Compiler/Main.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Compiler/Main.py 2022-03-24 10:16:46.000000000 +0000 @@ -9,8 +9,8 @@ import sys import io -if sys.version_info[:2] < (2, 6) or (3, 0) <= sys.version_info[:2] < (3, 2): - sys.stderr.write("Sorry, Cython requires Python 2.6+ or 3.2+, found %d.%d\n" % tuple(sys.version_info[:2])) +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 3): + sys.stderr.write("Sorry, Cython requires Python 2.7 or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) sys.exit(1) try: @@ -18,39 +18,41 @@ except ImportError: basestring = str -from . import Errors # Do not import Parsing here, import it when needed, because Parsing imports # Nodes, which globally needs debug command line options initialized to set a # conditional metaclass. These options are processed by CmdLine called from # main() in this file. # import Parsing +from . import Errors from .StringEncoding import EncodedString from .Scanning import PyrexScanner, FileSourceDescriptor from .Errors import PyrexError, CompileError, error, warning from .Symtab import ModuleScope from .. import Utils from . import Options +from .Options import CompilationOptions, default_options +from .CmdLine import parse_command_line +from .Lexicon import (unicode_start_ch_any, unicode_continuation_ch_any, + unicode_start_ch_range, unicode_continuation_ch_range) + + +def _make_range_re(chrs): + out = [] + for i in range(0, len(chrs), 2): + out.append(u"{0}-{1}".format(chrs[i], chrs[i+1])) + return u"".join(out) + +# py2 version looked like r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$" +module_name_pattern = u"[{0}{1}][{0}{2}{1}{3}]*".format( + unicode_start_ch_any, _make_range_re(unicode_start_ch_range), + unicode_continuation_ch_any, + _make_range_re(unicode_continuation_ch_range)) +module_name_pattern = re.compile(u"{0}(\\.{0})*$".format(module_name_pattern)) -from . import Version # legacy import needed by old PyTables versions -version = Version.version # legacy attribute - use "Cython.__version__" instead -module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") +standard_include_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.dirname(__file__)), 'Includes')) -verbose = 0 - -class CompilationData(object): - # Bundles the information that is passed from transform to transform. - # (For now, this is only) - - # While Context contains every pxd ever loaded, path information etc., - # this only contains the data related to a single compilation pass - # - # pyx ModuleNode Main code tree of this compilation. - # pxds {string : ModuleNode} Trees for the pxds used in the pyx. - # codewriter CCodeWriter Where to output final code. - # options CompilationOptions - # result CompilationResult - pass class Context(object): # This class encapsulates the context needed for compiling @@ -65,9 +67,10 @@ # language_level int currently 2 or 3 for Python 2/3 cython_scope = None + language_level = None # warn when not set but default to Py2 def __init__(self, include_directories, compiler_directives, cpp=False, - language_level=2, options=None, create_testscope=True): + language_level=None, options=None): # cython_scope is a hack, set to False by subclasses, in order to break # an infinite loop. # Better code organization would fix it. @@ -85,19 +88,30 @@ self.pxds = {} # full name -> node tree self._interned = {} # (type(value), value, *key_args) -> interned_value - standard_include_path = os.path.abspath(os.path.normpath( - os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) - self.include_directories = include_directories + [standard_include_path] - - self.set_language_level(language_level) + if language_level is not None: + self.set_language_level(language_level) self.gdb_debug_outputwriter = None + @classmethod + def from_options(cls, options): + return cls(options.include_path, options.compiler_directives, + options.cplus, options.language_level, options=options) + def set_language_level(self, level): + from .Future import print_function, unicode_literals, absolute_import, division, generator_stop + future_directives = set() + if level == '3str': + level = 3 + else: + level = int(level) + if level >= 3: + future_directives.add(unicode_literals) + if level >= 3: + future_directives.update([print_function, absolute_import, division, generator_stop]) self.language_level = level + self.future_directives = future_directives if level >= 3: - from .Future import print_function, unicode_literals, absolute_import, division - self.future_directives.update([print_function, unicode_literals, absolute_import, division]) self.modules['builtins'] = self.modules['__builtin__'] def intern_ustring(self, value, encoding=None): @@ -112,15 +126,6 @@ self._interned[key] = value return value - def intern_value(self, value, *key): - key = (type(value), value) + key - try: - return self._interned[key] - except KeyError: - pass - self._interned[key] = value - return value - # pipeline creation functions can now be found in Pipeline.py def process_pxd(self, source_desc, scope, module_name): @@ -168,7 +173,7 @@ if not module_name_pattern.match(qualified_name): raise CompileError(pos or (module_name, 0, 0), - "'%s' is not a valid module name" % module_name) + u"'%s' is not a valid module name" % module_name) if relative_to: if debug_find_module: @@ -204,8 +209,9 @@ # Set pxd_file_loaded such that we don't need to # look for the non-existing pxd file next time. scope.pxd_file_loaded = True - package_pathname = self.search_include_directories(qualified_name, ".py", pos) - if package_pathname and package_pathname.endswith('__init__.py'): + package_pathname = self.search_include_directories( + qualified_name, suffix=".py", source_pos=pos) + if package_pathname and package_pathname.endswith(Utils.PACKAGE_FILES): pass else: error(pos, "'%s.pxd' not found" % qualified_name.replace('.', os.sep)) @@ -227,7 +233,7 @@ pass return scope - def find_pxd_file(self, qualified_name, pos, sys_path=True): + def find_pxd_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): # Search include path (and sys.path if sys_path is True) for # the .pxd file corresponding to the given fully-qualified # module name. @@ -236,48 +242,36 @@ # the directory containing the source file is searched first # for a dotted filename, and its containing package root # directory is searched first for a non-dotted filename. - pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=sys_path) - if pxd is None: # XXX Keep this until Includes/Deprecated is removed - if (qualified_name.startswith('python') or - qualified_name in ('stdlib', 'stdio', 'stl')): - standard_include_path = os.path.abspath(os.path.normpath( - os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) - deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') - self.include_directories.append(deprecated_include_path) - try: - pxd = self.search_include_directories(qualified_name, ".pxd", pos) - finally: - self.include_directories.pop() - if pxd: - name = qualified_name - if name.startswith('python'): - warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1) - elif name in ('stdlib', 'stdio'): - warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1) - elif name in ('stl'): - warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1) + pxd = self.search_include_directories( + qualified_name, suffix=".pxd", source_pos=pos, sys_path=sys_path, source_file_path=source_file_path) if pxd is None and Options.cimport_from_pyx: return self.find_pyx_file(qualified_name, pos) return pxd - def find_pyx_file(self, qualified_name, pos): + def find_pyx_file(self, qualified_name, pos=None, source_file_path=None): # Search include path for the .pyx file corresponding to the # given fully-qualified module name, as for find_pxd_file(). - return self.search_include_directories(qualified_name, ".pyx", pos) + return self.search_include_directories( + qualified_name, suffix=".pyx", source_pos=pos, source_file_path=source_file_path) - def find_include_file(self, filename, pos): + def find_include_file(self, filename, pos=None, source_file_path=None): # Search list of include directories for filename. # Reports an error and returns None if not found. - path = self.search_include_directories(filename, "", pos, - include=True) + path = self.search_include_directories( + filename, source_pos=pos, include=True, source_file_path=source_file_path) if not path: error(pos, "'%s' not found" % filename) return path - def search_include_directories(self, qualified_name, suffix, pos, - include=False, sys_path=False): - return Utils.search_include_directories( - tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path) + def search_include_directories(self, qualified_name, + suffix=None, source_pos=None, include=False, sys_path=False, source_file_path=None): + include_dirs = self.include_directories + if sys_path: + include_dirs = include_dirs + sys.path + # include_dirs must be hashable for caching in @cached_function + include_dirs = tuple(include_dirs + [standard_include_path]) + return search_include_directories( + include_dirs, qualified_name, suffix or "", source_pos, include, source_file_path) def find_root_package_dir(self, file_path): return Utils.find_root_package_dir(file_path) @@ -291,15 +285,14 @@ c_time = Utils.modification_time(output_path) if Utils.file_newer_than(source_path, c_time): return 1 - pos = [source_path] pxd_path = Utils.replace_suffix(source_path, ".pxd") if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): return 1 for kind, name in self.read_dependency_file(source_path): if kind == "cimport": - dep_path = self.find_pxd_file(name, pos) + dep_path = self.find_pxd_file(name, source_file_path=source_path) elif kind == "include": - dep_path = self.search_include_directories(name, pos) + dep_path = self.search_include_directories(name, source_file_path=source_path) else: continue if dep_path and Utils.file_newer_than(dep_path, c_time): @@ -344,7 +337,7 @@ source_filename = source_desc.filename scope.cpp = self.cpp # Parse the given source file and return a parse tree. - num_errors = Errors.num_errors + num_errors = Errors.get_errors_count() try: with Utils.open_source_file(source_filename) as f: from . import Parsing @@ -356,14 +349,14 @@ from ..Parser import ConcreteSyntaxTree except ImportError: raise RuntimeError( - "Formal grammer can only be used with compiled Cython with an available pgen.") + "Formal grammar can only be used with compiled Cython with an available pgen.") ConcreteSyntaxTree.p_module(source_filename) except UnicodeDecodeError as e: #import traceback #traceback.print_exc() raise self._report_decode_error(source_desc, e) - if Errors.num_errors > num_errors: + if Errors.get_errors_count() > num_errors: raise CompileError() return tree @@ -403,20 +396,19 @@ return ".".join(names) def setup_errors(self, options, result): - Errors.reset() # clear any remaining error state + Errors.init_thread() if options.use_listing_file: path = result.listing_file = Utils.replace_suffix(result.main_source_file, ".lis") else: path = None - Errors.open_listing_file(path=path, - echo_to_stderr=options.errors_to_stderr) + Errors.open_listing_file(path=path, echo_to_stderr=options.errors_to_stderr) def teardown_errors(self, err, options, result): source_desc = result.compilation_source.source_desc if not isinstance(source_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") Errors.close_listing_file() - result.num_errors = Errors.num_errors + result.num_errors = Errors.get_errors_count() if result.num_errors > 0: err = True if err and result.c_file: @@ -426,6 +418,7 @@ pass result.c_file = None + def get_output_filename(source_filename, cwd, options): if options.cplus: c_suffix = ".cpp" @@ -441,6 +434,7 @@ else: return suggested_file_name + def create_default_resultobj(compilation_source, options): result = CompilationResult() result.main_source_file = compilation_source.source_desc.filename @@ -451,23 +445,33 @@ result.embedded_metadata = options.embedded_metadata return result + def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline + # ensure that the inputs are unicode (for Python 2) + if sys.version_info[0] == 2: + source = Utils.decode_filename(source) + if full_module_name: + full_module_name = Utils.decode_filename(full_module_name) + source_ext = os.path.splitext(source)[1] - options.configure_language_defaults(source_ext[1:]) # py/pyx + options.configure_language_defaults(source_ext[1:]) # py/pyx if context is None: - context = options.create_context() + context = Context.from_options(options) # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) full_module_name = full_module_name or context.extract_module_name(source, options) + full_module_name = EncodedString(full_module_name) + + Utils.raise_error_if_module_name_forbidden(full_module_name) if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): - rel_path = source # safety measure to prevent printing incorrect paths + rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path source_desc = FileSourceDescriptor(abs_path, rel_path) @@ -481,8 +485,7 @@ html_filename = os.path.splitext(result.c_file)[0] + ".html" if os.path.exists(html_filename): with io.open(html_filename, "r", encoding="UTF-8") as html_file: - line = html_file.readline() - if line.startswith(u' State %d\n" % (key, state['number'])) @@ -229,6 +215,7 @@ if state: file.write(" %s --> State %d\n" % (key, state['number'])) + @cython.locals(char_list=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, c1=cython.long, c2=cython.long) def chars_to_ranges(self, char_list): char_list.sort() i = 0 diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Regexps.py cython-0.20.1+1~202203241016-9537/Cython/Plex/Regexps.py --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Regexps.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Regexps.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,21 +1,16 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Regular Expressions -# -#======================================================================= +""" +Python Lexical Analyser +Regular Expressions +""" from __future__ import absolute_import import types -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint from . import Errors +maxint = 2**31-1 # sentinel value + # # Constants # @@ -186,37 +181,6 @@ # These are the basic REs from which all others are built. # -## class Char(RE): -## """ -## Char(c) is an RE which matches the character |c|. -## """ - -## nullable = 0 - -## def __init__(self, char): -## self.char = char -## self.match_nl = char == '\n' - -## def build_machine(self, m, initial_state, final_state, match_bol, nocase): -## c = self.char -## if match_bol and c != BOL: -## s1 = self.build_opt(m, initial_state, BOL) -## else: -## s1 = initial_state -## if c == '\n' or c == EOF: -## s1 = self.build_opt(m, s1, EOL) -## if len(c) == 1: -## code = ord(self.char) -## s1.add_transition((code, code+1), final_state) -## if nocase and is_letter_code(code): -## code2 = other_case_code(code) -## s1.add_transition((code2, code2+1), final_state) -## else: -## s1.add_transition(c, final_state) - -## def calc_str(self): -## return "Char(%s)" % repr(self.char) - def Char(c): """ @@ -428,6 +392,7 @@ name = "Case" return "%s(%s)" % (name, self.re) + # # Composite RE constructors # ------------------------- @@ -469,7 +434,6 @@ """ Any(s) is an RE which matches any character in the string |s|. """ - #result = apply(Alt, tuple(map(Char, s))) result = CodeRanges(chars_to_ranges(s)) result.str = "Any(%s)" % repr(s) return result @@ -549,6 +513,7 @@ """ return SwitchCase(re, nocase=0) + # # RE Constants # @@ -573,4 +538,3 @@ Eof is an RE which matches the end of the file. """ Eof.str = "Eof" - diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Scanners.pxd cython-0.20.1+1~202203241016-9537/Cython/Plex/Scanners.pxd --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Scanners.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Scanners.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -29,17 +29,18 @@ cdef public level @cython.locals(input_state=long) - cdef next_char(self) + cdef inline next_char(self) @cython.locals(action=Action) cpdef tuple read(self) - cdef tuple scan_a_token(self) - cdef tuple position(self) + cdef inline tuple scan_a_token(self) + ##cdef tuple position(self) # used frequently by Parsing.py - @cython.locals(cur_pos=long, cur_line=long, cur_line_start=long, - input_state=long, next_pos=long, state=dict, - buf_start_pos=long, buf_len=long, buf_index=long, - trace=bint, discard=long, data=unicode, buffer=unicode) + @cython.final + @cython.locals(cur_pos=Py_ssize_t, cur_line=Py_ssize_t, cur_line_start=Py_ssize_t, + input_state=long, next_pos=Py_ssize_t, state=dict, + buf_start_pos=Py_ssize_t, buf_len=Py_ssize_t, buf_index=Py_ssize_t, + trace=bint, discard=Py_ssize_t, data=unicode, buffer=unicode) cdef run_machine_inlined(self) - cdef begin(self, state) - cdef produce(self, value, text = *) + cdef inline begin(self, state) + cdef inline produce(self, value, text = *) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Scanners.py cython-0.20.1+1~202203241016-9537/Cython/Plex/Scanners.py --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Scanners.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Scanners.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,17 +1,15 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# -# Scanning an input stream -# -#======================================================================= +# cython: language_level=3str +# cython: auto_pickle=False +""" +Python Lexical Analyser +Scanning an input stream +""" from __future__ import absolute_import import cython -cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) +cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) # noqa:E402 from . import Errors from .Regexps import BOL, EOL, EOF @@ -172,26 +170,28 @@ buf_len = len(buffer) b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ None, 0, 0, 0, u'', 0, 0 + trace = self.trace while 1: - if trace: #TRACE# - print("State %d, %d/%d:%s -->" % ( #TRACE# - state['number'], input_state, cur_pos, repr(cur_char))) #TRACE# + if trace: + print("State %d, %d/%d:%s -->" % ( + state['number'], input_state, cur_pos, repr(cur_char))) + # Begin inlined self.save_for_backup() - #action = state.action #@slow - action = state['action'] #@fast + action = state['action'] if action is not None: b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos # End inlined self.save_for_backup() + c = cur_char - #new_state = state.new_state(c) #@slow - new_state = state.get(c, NOT_FOUND) #@fast - if new_state is NOT_FOUND: #@fast - new_state = c and state.get('else') #@fast + new_state = state.get(c, NOT_FOUND) + if new_state is NOT_FOUND: + new_state = c and state.get('else') + if new_state: - if trace: #TRACE# - print("State %d" % new_state['number']) #TRACE# + if trace: + print("State %d" % new_state['number']) state = new_state # Begin inlined: self.next_char() if input_state == 1: @@ -239,8 +239,8 @@ cur_char = u'' # End inlined self.next_char() else: # not new_state - if trace: #TRACE# - print("blocked") #TRACE# + if trace: + print("blocked") # Begin inlined: action = self.back_up() if b_action is not None: (action, cur_pos, cur_line, cur_line_start, @@ -251,15 +251,16 @@ action = None break # while 1 # End inlined: action = self.back_up() + self.cur_pos = cur_pos self.cur_line = cur_line self.cur_line_start = cur_line_start self.cur_char = cur_char self.input_state = input_state self.next_pos = next_pos - if trace: #TRACE# - if action is not None: #TRACE# - print("Doing %s" % action) #TRACE# + if trace: + if action is not None: + print("Doing %s" % action) return action def next_char(self): @@ -291,7 +292,7 @@ else: # input_state = 5 self.cur_char = u'' if self.trace: - print("--> [%d] %d %s" % (input_state, self.cur_pos, repr(self.cur_char))) + print("--> [%d] %d %r" % (input_state, self.cur_pos, self.cur_char)) def position(self): """ @@ -305,7 +306,8 @@ return (self.name, self.start_line, self.start_col) def get_position(self): - """Python accessible wrapper around position(), only for error reporting. + """ + Python accessible wrapper around position(), only for error reporting. """ return self.position() @@ -335,3 +337,4 @@ Override this method if you want something to be done at end of file. """ + pass diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Timing.py cython-0.20.1+1~202203241016-9537/Cython/Plex/Timing.py --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Timing.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Timing.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,23 +0,0 @@ -# -# Get time in platform-dependent way -# - -from __future__ import absolute_import - -import os -from sys import platform, exit, stderr - -if platform == 'mac': - import MacOS - def time(): - return MacOS.GetTicks() / 60.0 - timekind = "real" -elif hasattr(os, 'times'): - def time(): - t = os.times() - return t[0] + t[1] - timekind = "cpu" -else: - stderr.write( - "Don't know how to get time on platform %s\n" % repr(platform)) - exit(1) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Traditional.py cython-0.20.1+1~202203241016-9537/Cython/Plex/Traditional.py --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Traditional.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Traditional.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,158 +0,0 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Traditional Regular Expression Syntax -# -#======================================================================= - -from __future__ import absolute_import - -from .Regexps import Alt, Seq, Rep, Rep1, Opt, Any, AnyBut, Bol, Eol, Char -from .Errors import PlexError - - -class RegexpSyntaxError(PlexError): - pass - - -def re(s): - """ - Convert traditional string representation of regular expression |s| - into Plex representation. - """ - return REParser(s).parse_re() - - -class REParser(object): - def __init__(self, s): - self.s = s - self.i = -1 - self.end = 0 - self.next() - - def parse_re(self): - re = self.parse_alt() - if not self.end: - self.error("Unexpected %s" % repr(self.c)) - return re - - def parse_alt(self): - """Parse a set of alternative regexps.""" - re = self.parse_seq() - if self.c == '|': - re_list = [re] - while self.c == '|': - self.next() - re_list.append(self.parse_seq()) - re = Alt(*re_list) - return re - - def parse_seq(self): - """Parse a sequence of regexps.""" - re_list = [] - while not self.end and not self.c in "|)": - re_list.append(self.parse_mod()) - return Seq(*re_list) - - def parse_mod(self): - """Parse a primitive regexp followed by *, +, ? modifiers.""" - re = self.parse_prim() - while not self.end and self.c in "*+?": - if self.c == '*': - re = Rep(re) - elif self.c == '+': - re = Rep1(re) - else: # self.c == '?' - re = Opt(re) - self.next() - return re - - def parse_prim(self): - """Parse a primitive regexp.""" - c = self.get() - if c == '.': - re = AnyBut("\n") - elif c == '^': - re = Bol - elif c == '$': - re = Eol - elif c == '(': - re = self.parse_alt() - self.expect(')') - elif c == '[': - re = self.parse_charset() - self.expect(']') - else: - if c == '\\': - c = self.get() - re = Char(c) - return re - - def parse_charset(self): - """Parse a charset. Does not include the surrounding [].""" - char_list = [] - invert = 0 - if self.c == '^': - invert = 1 - self.next() - if self.c == ']': - char_list.append(']') - self.next() - while not self.end and self.c != ']': - c1 = self.get() - if self.c == '-' and self.lookahead(1) != ']': - self.next() - c2 = self.get() - for a in range(ord(c1), ord(c2) + 1): - char_list.append(chr(a)) - else: - char_list.append(c1) - chars = ''.join(char_list) - if invert: - return AnyBut(chars) - else: - return Any(chars) - - def next(self): - """Advance to the next char.""" - s = self.s - i = self.i = self.i + 1 - if i < len(s): - self.c = s[i] - else: - self.c = '' - self.end = 1 - - def get(self): - if self.end: - self.error("Premature end of string") - c = self.c - self.next() - return c - - def lookahead(self, n): - """Look ahead n chars.""" - j = self.i + n - if j < len(self.s): - return self.s[j] - else: - return '' - - def expect(self, c): - """ - Expect to find character |c| at current position. - Raises an exception otherwise. - """ - if self.c == c: - self.next() - else: - self.error("Missing %s" % repr(c)) - - def error(self, mess): - """Raise exception to signal syntax error in regexp.""" - raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( - repr(self.s), self.i, mess)) - - - diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Transitions.pxd cython-0.20.1+1~202203241016-9537/Cython/Plex/Transitions.pxd --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Transitions.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Transitions.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +cimport cython + +cdef long maxint + +@cython.final +cdef class TransitionMap: + cdef list map + cdef dict special + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add(self, event, new_state) + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add_set(self, event, new_set) + + @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) + cpdef iteritems(self) + + @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t) + cdef split(self, long code) + + cdef get_special(self, event) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Plex/Transitions.py cython-0.20.1+1~202203241016-9537/Cython/Plex/Transitions.py --- cython-0.20.1+1~201611251650-6686/Cython/Plex/Transitions.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Plex/Transitions.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,15 +1,11 @@ -# -# Plex - Transition Maps -# -# This version represents state sets directly as dicts for speed. -# +# cython: auto_pickle=False +""" +Plex - Transition Maps -from __future__ import absolute_import +This version represents state sets directly as dicts for speed. +""" -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value class TransitionMap(object): @@ -40,24 +36,19 @@ kept separately in a dictionary. """ - map = None # The list of codes and states - special = None # Mapping for special events - def __init__(self, map=None, special=None): if not map: map = [-maxint, {}, maxint] if not special: special = {} - self.map = map - self.special = special - #self.check() ### + self.map = map # The list of codes and states + self.special = special # Mapping for special events - def add(self, event, new_state, - TupleType=tuple): + def add(self, event, new_state): """ Add transition to |new_state| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -68,12 +59,11 @@ else: self.get_special(event)[new_state] = 1 - def add_set(self, event, new_set, - TupleType=tuple): + def add_set(self, event, new_set): """ Add transitions to the states in |new_set| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -84,15 +74,13 @@ else: self.get_special(event).update(new_set) - def get_epsilon(self, - none=None): + def get_epsilon(self): """ Return the mapping for epsilon, or None. """ - return self.special.get('', none) + return self.special.get('') - def iteritems(self, - len=len): + def iteritems(self): """ Return the mapping as an iterable of ((code1, code2), state_set) and (special_event, state_set) pairs. @@ -119,8 +107,7 @@ # ------------------- Private methods -------------------- - def split(self, code, - len=len, maxint=maxint): + def split(self, code): """ Search the list for the position of the split point for |code|, inserting a new split point if necessary. Returns index |i| such @@ -132,6 +119,7 @@ # Special case: code == map[-1] if code == maxint: return hi + # General case lo = 0 # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 @@ -147,7 +135,6 @@ return lo else: map[hi:hi] = [code, map[hi - 1].copy()] - #self.check() ### return hi def get_special(self, event): @@ -243,9 +230,5 @@ # State set manipulation functions # -#def merge_state_sets(set1, set2): -# for state in set2.keys(): -# set1[state] = 1 - def state_set_str(set): return "[%s]" % ','.join(["S%d" % state.number for state in set]) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Runtime/refnanny.pyx cython-0.20.1+1~202203241016-9537/Cython/Runtime/refnanny.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Runtime/refnanny.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Runtime/refnanny.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ -# cython: language_level=3 +# cython: language_level=3, auto_pickle=False -from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF, Py_XINCREF +from cpython.ref cimport PyObject, Py_INCREF, Py_CLEAR, Py_XDECREF, Py_XINCREF from cpython.exc cimport PyErr_Fetch, PyErr_Restore from cpython.pystate cimport PyThreadState_Get @@ -10,6 +10,9 @@ reflog = [] cdef log(level, action, obj, lineno): + if reflog is None: + # can happen during finalisation + return if loglevel >= level: reflog.append((lineno, action, id(obj))) @@ -29,7 +32,7 @@ self.refs = {} # id -> (count, [lineno]) self.errors = [] - cdef regref(self, obj, lineno, bint is_null): + cdef regref(self, obj, Py_ssize_t lineno, bint is_null): log(LOG_ALL, u'regref', u"" if is_null else obj, lineno) if is_null: self.errors.append(f"NULL argument on line {lineno}") @@ -39,7 +42,7 @@ self.refs[id_] = (count + 1, linenumbers) linenumbers.append(lineno) - cdef bint delref(self, obj, lineno, bint is_null) except -1: + cdef bint delref(self, obj, Py_ssize_t lineno, bint is_null) except -1: # returns whether it is ok to do the decref operation log(LOG_ALL, u'delref', u"" if is_null else obj, lineno) if is_null: @@ -50,12 +53,11 @@ if count == 0: self.errors.append(f"Too many decrefs on line {lineno}, reference acquired on lines {linenumbers!r}") return False - elif count == 1: + if count == 1: del self.refs[id_] - return True else: self.refs[id_] = (count - 1, linenumbers) - return True + return True cdef end(self): if self.refs: @@ -63,121 +65,117 @@ for count, linenos in self.refs.itervalues(): msg += f"\n ({count}) acquired on lines: {u', '.join([f'{x}' for x in linenos])}" self.errors.append(msg) - if self.errors: - return u"\n".join([u'REFNANNY: '+error for error in self.errors]) - else: - return None + return u"\n".join([f'REFNANNY: {error}' for error in self.errors]) if self.errors else None + -cdef void report_unraisable(object e=None): +cdef void report_unraisable(filename, Py_ssize_t lineno, object e=None): try: if e is None: import sys e = sys.exc_info()[1] - print(f"refnanny raised an exception: {e}") - except: - pass # We absolutely cannot exit with an exception + print(f"refnanny raised an exception from {filename}:{lineno}: {e}") + finally: + return # We absolutely cannot exit with an exception + # All Python operations must happen after any existing # exception has been fetched, in case we are called from # exception-handling code. -cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except NULL: +cdef PyObject* SetupContext(char* funcname, Py_ssize_t lineno, char* filename) except NULL: if Context is None: # Context may be None during finalize phase. # In that case, we don't want to be doing anything fancy # like caching and resetting exceptions. return NULL cdef (PyObject*) type = NULL, value = NULL, tb = NULL, result = NULL - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: ctx = Context(funcname, lineno, filename) Py_INCREF(ctx) result = ctx except Exception, e: - report_unraisable(e) + report_unraisable(filename, lineno, e) PyErr_Restore(type, value, tb) return result -cdef void GOTREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GOTREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - (ctx).regref(None, lineno, True) - else: - (ctx).regref(p_obj, lineno, False) - except: - report_unraisable() + (ctx).regref( + p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) + report_unraisable((ctx).filename, lineno=(ctx).start) + finally: + PyErr_Restore(type, value, tb) + return # swallow any exceptions -cdef int GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, int lineno): +cdef bint GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return 1 cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef bint decref_ok = False PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - decref_ok = (ctx).delref(None, lineno, True) - else: - decref_ok = (ctx).delref(p_obj, lineno, False) - except: - report_unraisable() + decref_ok = (ctx).delref( + p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) - return decref_ok + report_unraisable((ctx).filename, lineno=(ctx).start) + finally: + PyErr_Restore(type, value, tb) + return decref_ok # swallow any exceptions -cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): GIVEREF_and_report(ctx, p_obj, lineno) -cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void INCREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): Py_XINCREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL GOTREF(ctx, obj, lineno) -cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void DECREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): if GIVEREF_and_report(ctx, obj, lineno): Py_XDECREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL cdef void FinishContext(PyObject** ctx): if ctx == NULL or ctx[0] == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef object errors = None cdef Context context - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: - try: - context = ctx[0] - errors = context.end() - if errors: - print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") - print(errors) - context = None - except: - report_unraisable() + context = ctx[0] + errors = context.end() + if errors: + print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") + print(errors) + context = None except: - # __Pyx_GetException may itself raise errors - pass - Py_XDECREF(ctx[0]) - ctx[0] = NULL - PyErr_Restore(type, value, tb) + report_unraisable( + context.filename if context is not None else None, + lineno=context.start if context is not None else 0, + ) + finally: + Py_CLEAR(ctx[0]) + PyErr_Restore(type, value, tb) + return # swallow any exceptions ctypedef struct RefNannyAPIStruct: - void (*INCREF)(PyObject*, PyObject*, int) - void (*DECREF)(PyObject*, PyObject*, int) - void (*GOTREF)(PyObject*, PyObject*, int) - void (*GIVEREF)(PyObject*, PyObject*, int) - PyObject* (*SetupContext)(char*, int, char*) except NULL + void (*INCREF)(PyObject*, PyObject*, Py_ssize_t) + void (*DECREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GOTREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GIVEREF)(PyObject*, PyObject*, Py_ssize_t) + PyObject* (*SetupContext)(char*, Py_ssize_t, char*) except NULL void (*FinishContext)(PyObject**) cdef RefNannyAPIStruct api diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Shadow.py cython-0.20.1+1~202203241016-9537/Cython/Shadow.py --- cython-0.20.1+1~201611251650-6686/Cython/Shadow.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Shadow.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,7 @@ # cython.* namespace for pure mode. from __future__ import absolute_import -__version__ = "0.25b0" +__version__ = "3.0.0a10" try: from __builtin__ import basestring @@ -71,7 +71,7 @@ else: # int[8] etc. assert int(item) == item # array size must be a plain integer - array(base_type, item) + return array(base_type, item) # END shameless copy @@ -108,17 +108,22 @@ cclass = ccall = cfunc = _EmptyDecoratorAndManager() returns = wraparound = boundscheck = initializedcheck = nonecheck = \ - overflowcheck = embedsignature = cdivision = cdivision_warnings = \ - always_allows_keywords = profile = linetrace = infer_type = \ + embedsignature = cdivision = cdivision_warnings = \ + always_allows_keywords = profile = linetrace = infer_types = \ unraisable_tracebacks = freelist = \ - lambda arg: _EmptyDecoratorAndManager() + lambda _: _EmptyDecoratorAndManager() -optimization = _Optimization() +exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager() -overflowcheck.fold = optimization.use_switch = \ - optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() +overflowcheck = lambda _: _EmptyDecoratorAndManager() +optimize = _Optimization() -final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator +overflowcheck.fold = optimize.use_switch = \ + optimize.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() + +final = internal = type_version_tag = no_gc_clear = no_gc = total_ordering = _empty_decorator + +binding = lambda _: _empty_decorator _cython_inline = None @@ -141,26 +146,33 @@ # Special functions def cdiv(a, b): - q = a / b - if q < 0: - q += 1 + if a < 0: + a = -a + b = -b + if b < 0: + return (a + b + 1) // b + return a // b def cmod(a, b): r = a % b - if (a*b) < 0: + if (a * b) < 0 and r: r -= b return r # Emulated language constructs -def cast(type, *args, **kwargs): +def cast(t, *args, **kwargs): kwargs.pop('typecheck', None) assert not kwargs - if hasattr(type, '__call__'): - return type(*args) - else: - return args[0] + + if isinstance(t, typedef): + return t(*args) + elif isinstance(t, type): # Doesn't work with old-style classes of Python 2.x + if len(args) != 1 or not (args[0] is None or isinstance(args[0], t)): + return t(*args) + + return args[0] def sizeof(arg): return 1 @@ -172,18 +184,30 @@ def address(arg): return pointer(type(arg))([arg]) -def declare(type=None, value=_Unspecified, **kwds): - if type not in (None, object) and hasattr(type, '__call__'): - if value is not _Unspecified: - return type(value) - else: - return type() +def _is_value_type(t): + if isinstance(t, typedef): + return _is_value_type(t._basetype) + + return isinstance(t, type) and issubclass(t, (StructType, UnionType, ArrayType)) + +def declare(t=None, value=_Unspecified, **kwds): + if value is not _Unspecified: + return cast(t, value) + elif _is_value_type(t): + return t() else: - return value + return None class _nogil(object): - """Support for 'with nogil' statement + """Support for 'with nogil' statement and @nogil decorator. """ + def __call__(self, x): + if callable(x): + # Used as function decorator => return the function unchanged. + return x + # Used as conditional context manager or to create an "@nogil(True/False)" decorator => keep going. + return self + def __enter__(self): pass def __exit__(self, exc_class, exc, tb): @@ -193,6 +217,7 @@ gil = _nogil() del _nogil + # Emulated types class CythonMetaType(type): @@ -244,24 +269,45 @@ class ArrayType(PointerType): - def __init__(self): - self._items = [None] * self._n + def __init__(self, value=None): + if value is None: + self._items = [None] * self._n + else: + super(ArrayType, self).__init__(value) class StructType(CythonType): - def __init__(self, cast_from=_Unspecified, **data): - if cast_from is not _Unspecified: - # do cast - if len(data) > 0: - raise ValueError('Cannot accept keyword arguments when casting.') - if type(cast_from) is not type(self): - raise ValueError('Cannot cast from %s'%cast_from) - for key, value in cast_from.__dict__.items(): - setattr(self, key, value) + def __init__(self, *posargs, **data): + if not (posargs or data): + return + if posargs and data: + raise ValueError('Cannot accept both positional and keyword arguments.') + + # Allow 'cast_from' as single positional or keyword argument. + if data and len(data) == 1 and 'cast_from' in data: + cast_from = data.pop('cast_from') + elif len(posargs) == 1 and type(posargs[0]) is type(self): + cast_from, posargs = posargs[0], () + elif posargs: + for key, arg in zip(self._members, posargs): + setattr(self, key, arg) + return else: for key, value in data.items(): + if key not in self._members: + raise ValueError("Invalid struct attribute for %s: %s" % ( + self.__class__.__name__, key)) setattr(self, key, value) + return + + # do cast + if data: + raise ValueError('Cannot accept keyword arguments when casting.') + if type(cast_from) is not type(self): + raise ValueError('Cannot cast from %s' % cast_from) + for key, value in cast_from.__dict__.items(): + setattr(self, key, value) def __setattr__(self, key, value): if key in self._members: @@ -282,7 +328,7 @@ elif type(cast_from) is type(self): datadict = cast_from.__dict__ else: - raise ValueError('Cannot cast from %s'%cast_from) + raise ValueError('Cannot cast from %s' % cast_from) else: datadict = data if len(datadict) > 1: @@ -379,10 +425,34 @@ # Predefined types -int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t'] -float_types = ['longdouble', 'double', 'float'] -complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex'] -other_types = ['bint', 'void'] +int_types = [ + 'char', + 'short', + 'Py_UNICODE', + 'int', + 'Py_UCS4', + 'long', + 'longlong', + 'Py_hash_t', + 'Py_ssize_t', + 'size_t', +] +float_types = [ + 'longdouble', + 'double', + 'float', +] +complex_types = [ + 'longdoublecomplex', + 'doublecomplex', + 'floatcomplex', + 'complex', +] +other_types = [ + 'bint', + 'void', + 'Py_tss_t', +] to_repr = { 'longlong': 'long long', @@ -417,14 +487,17 @@ gs[name] = typedef(py_complex, to_repr(name, name)) bint = typedef(bool, "bint") -void = typedef(int, "void") +void = typedef(None, "void") +Py_tss_t = typedef(None, "Py_tss_t") for t in int_types + float_types + complex_types + other_types: for i in range(1, 4): - gs["%s_%s" % ('p'*i, t)] = globals()[t]._pointer(i) + gs["%s_%s" % ('p'*i, t)] = gs[t]._pointer(i) -void = typedef(None, "void") -NULL = p_void(0) +NULL = gs['p_void'](0) + +# looks like 'gs' has some users out there by now... +#del gs integral = floating = numeric = _FusedType() @@ -440,7 +513,7 @@ def parallel(self, num_threads=None): return nogil - def prange(self, start=0, stop=None, step=1, schedule=None, nogil=False): + def prange(self, start=0, stop=None, step=1, nogil=False, schedule=None, chunksize=None, num_threads=None): if stop is None: stop = start start = 0 @@ -452,6 +525,53 @@ # def threadsavailable(self): # return 1 -import sys +class CythonDotImportedFromElsewhere(object): + """ + cython.dataclasses just shadows the standard library modules of the same name + """ + def __init__(self, module): + self.__path__ = [] + self.__file__ = None + self.__name__ = module + self.__package__ = module + + def __getattr__(self, attr): + # we typically only expect this to be called once + from importlib import import_module + import sys + try: + mod = import_module(self.__name__) + except ImportError: + # but if they don't exist (Python is not sufficiently up-to-date) then + # you can't use them + raise AttributeError("%s: the standard library module %s is not available" % + (attr, self.__name__)) + sys.modules['cython.%s' % self.__name__] = mod + return getattr(mod, attr) + + +class CythonCImports(object): + """ + Simplistic module mock to make cimports sort-of work in Python code. + """ + def __init__(self, module): + self.__path__ = [] + self.__file__ = None + self.__name__ = module + self.__package__ = module + + def __getattr__(self, item): + if item.startswith('__') and item.endswith('__'): + raise AttributeError(item) + return __import__(item) + + +import math, sys sys.modules['cython.parallel'] = CythonDotParallel() -del sys +sys.modules['cython.cimports'] = CythonCImports('cython.cimports') +sys.modules['cython.cimports.libc'] = CythonCImports('cython.cimports.libc') +sys.modules['cython.cimports.libc.math'] = math +# In pure Python mode @cython.dataclasses.dataclass and dataclass field should just +# shadow the standard library ones (if they are available) +dataclasses = sys.modules['cython.dataclasses'] = CythonDotImportedFromElsewhere('dataclasses') +del math, sys diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Shadow.pyi cython-0.20.1+1~202203241016-9537/Cython/Shadow.pyi --- cython-0.20.1+1~201611251650-6686/Cython/Shadow.pyi 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Shadow.pyi 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,102 @@ +from builtins import (int as py_int, float as py_float, + bool as py_bool, str as py_str, complex as py_complex) +from typing import (Union, Dict, Any, Sequence, Optional, + List, TypeVar, Type, Generic) + +int = py_int +long = py_int +longlong = py_int +short = py_int +char = py_int +sint = py_int +slong = py_int +slonglong = py_int +sshort = py_int +schar = py_int +uint = py_int +ulong = py_int +ulonglong = py_int +ushort = py_int +uchar = py_int +size_t = py_int +Py_ssize_t = py_int +Py_UCS4 = Union[py_int, str] +Py_UNICODE = Union[py_int, str] +float = py_float +double = py_float +longdouble = py_float +complex = py_complex +floatcomplex = py_complex +doublecomplex = py_complex +longdoublecomplex = py_complex +bint = py_bool +void = Union[None] +basestring = py_str +unicode = py_str + +gs: Dict[str, Any] # Should match the return type of globals() + +_T = TypeVar('_T') + +class _ArrayType(object, Generic[_T]): + is_array: bool + subtypes: Sequence[str] + dtype: _T + ndim: int + is_c_contig: bool + is_f_contig: bool + inner_contig: bool + broadcasting: Any + + # broadcasting is not used, so it's not clear about its type + def __init__(self, dtype: _T, ndim: int, is_c_contig: bool = ..., + is_f_contig: bool = ..., inner_contig: bool = ..., + broadcasting: Any = ...) -> None: ... + def __repr__(self) -> str: ... + +class CythonTypeObject(object): + ... + +class CythonType(CythonTypeObject): + ... + +class PointerType(CythonType, Generic[_T]): + def __init__( + self, + value: Optional[Union[ArrayType[_T], PointerType[_T], List[_T], int]] = ... + ) -> None: ... + def __getitem__(self, ix: int) -> _T: ... + def __setitem__(self, ix: int, value: _T) -> None: ... + def __eq__(self, value: object) -> bool: ... + def __repr__(self) -> str: ... + +class ArrayType(PointerType[_T]): + def __init__(self) -> None: ... + +#class StructType(CythonType, Generic[_T]): +# def __init__( +# self, +# value: List[Type[_T]] = ... +# ) -> None: ... + +def index_type( + base_type: _T, item: Union[tuple, slice, int]) -> _ArrayType[_T]: ... + +def pointer(basetype: _T) -> Type[PointerType[_T]]: ... + +def array(basetype: _T, n: int) -> Type[ArrayType[_T]]: ... + +#def struct(basetype: _T) -> Type[StructType[_T]]: ... + +class typedef(CythonType, Generic[_T]): + name: str + + def __init__(self, type: _T, name: Optional[str] = ...) -> None: ... + def __call__(self, *arg: Any) -> _T: ... + def __repr__(self) -> str: ... + __getitem__ = index_type + +#class _FusedType(CythonType, Generic[_T]): +# def __init__(self) -> None: ... + +#def fused_type(*args: Tuple[_T]) -> Type[FusedType[_T]]: ... diff -Nru cython-0.20.1+1~201611251650-6686/Cython/StringIOTree.pxd cython-0.20.1+1~202203241016-9537/Cython/StringIOTree.pxd --- cython-0.20.1+1~201611251650-6686/Cython/StringIOTree.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/StringIOTree.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +cimport cython + +cdef object StringIO + +@cython.final +cdef class StringIOTree: + cdef public list prepended_children + cdef public object stream + cdef public object write + cdef public list markers + + @cython.locals(x=StringIOTree) + cpdef getvalue(self) + @cython.locals(x=StringIOTree) + cdef _collect_in(self, list target_list) + @cython.locals(child=StringIOTree) + cpdef copyto(self, target) + cpdef commit(self) + #def insert(self, iotree) + #def insertion_point(self) + @cython.locals(c=StringIOTree) + cpdef allmarkers(self) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/StringIOTree.py cython-0.20.1+1~202203241016-9537/Cython/StringIOTree.py --- cython-0.20.1+1~201611251650-6686/Cython/StringIOTree.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/StringIOTree.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,47 @@ +# cython: auto_pickle=False + +r""" +Implements a buffer with insertion points. When you know you need to +"get back" to a place and write more later, simply call insertion_point() +at that spot and get a new StringIOTree object that is "left behind". + +EXAMPLE: + +>>> a = StringIOTree() +>>> _= a.write('first\n') +>>> b = a.insertion_point() +>>> _= a.write('third\n') +>>> _= b.write('second\n') +>>> a.getvalue().split() +['first', 'second', 'third'] + +>>> c = b.insertion_point() +>>> d = c.insertion_point() +>>> _= d.write('alpha\n') +>>> _= b.write('gamma\n') +>>> _= c.write('beta\n') +>>> b.getvalue().split() +['second', 'alpha', 'beta', 'gamma'] + +>>> try: from cStringIO import StringIO +... except ImportError: from io import StringIO + +>>> i = StringIOTree() +>>> d.insert(i) +>>> _= i.write('inserted\n') +>>> out = StringIO() +>>> a.copyto(out) +>>> out.getvalue().split() +['first', 'second', 'alpha', 'inserted', 'beta', 'gamma', 'third'] +""" + +from __future__ import absolute_import #, unicode_literals + try: + # Prefer cStringIO since io.StringIO() does not support writing 'str' in Py2. from cStringIO import StringIO except ImportError: - from io import StringIO # does not support writing 'str' in Py2 + from io import StringIO class StringIOTree(object): @@ -18,10 +58,17 @@ self.markers = [] def getvalue(self): - content = [x.getvalue() for x in self.prepended_children] - content.append(self.stream.getvalue()) + content = [] + self._collect_in(content) return "".join(content) + def _collect_in(self, target_list): + for x in self.prepended_children: + x._collect_in(target_list) + stream_content = self.stream.getvalue() + if stream_content: + target_list.append(stream_content) + def copyto(self, target): """Potentially cheaper than getvalue as no string concatenation needs to happen.""" @@ -70,34 +117,47 @@ children = self.prepended_children return [m for c in children for m in c.allmarkers()] + self.markers + """ + # Print the result of allmarkers in a nice human-readable form. Use it only for debugging. + # Prints e.g. + # /path/to/source.pyx: + # cython line 2 maps to 3299-3343 + # cython line 4 maps to 2236-2245 2306 3188-3201 + # /path/to/othersource.pyx: + # cython line 3 maps to 1234-1270 + # ... + # Note: In the example above, 3343 maps to line 2, 3344 does not. + def print_hr_allmarkers(self): + from collections import defaultdict + markers = self.allmarkers() + totmap = defaultdict(lambda: defaultdict(list)) + for c_lineno, (cython_desc, cython_lineno) in enumerate(markers): + if cython_lineno > 0 and cython_desc.filename is not None: + totmap[cython_desc.filename][cython_lineno].append(c_lineno + 1) + reprstr = "" + if totmap == 0: + reprstr += "allmarkers is empty\n" + try: + sorted(totmap.items()) + except: + print(totmap) + print(totmap.items()) + for cython_path, filemap in sorted(totmap.items()): + reprstr += cython_path + ":\n" + for cython_lineno, c_linenos in sorted(filemap.items()): + reprstr += "\tcython line " + str(cython_lineno) + " maps to " + i = 0 + while i < len(c_linenos): + reprstr += str(c_linenos[i]) + flag = False + while i+1 < len(c_linenos) and c_linenos[i+1] == c_linenos[i]+1: + i += 1 + flag = True + if flag: + reprstr += "-" + str(c_linenos[i]) + " " + i += 1 + reprstr += "\n" -__doc__ = r""" -Implements a buffer with insertion points. When you know you need to -"get back" to a place and write more later, simply call insertion_point() -at that spot and get a new StringIOTree object that is "left behind". - -EXAMPLE: - ->>> a = StringIOTree() ->>> _= a.write('first\n') ->>> b = a.insertion_point() ->>> _= a.write('third\n') ->>> _= b.write('second\n') ->>> a.getvalue().split() -['first', 'second', 'third'] - ->>> c = b.insertion_point() ->>> d = c.insertion_point() ->>> _= d.write('alpha\n') ->>> _= b.write('gamma\n') ->>> _= c.write('beta\n') ->>> b.getvalue().split() -['second', 'alpha', 'beta', 'gamma'] ->>> i = StringIOTree() ->>> d.insert(i) ->>> _= i.write('inserted\n') ->>> out = StringIO() ->>> a.copyto(out) ->>> out.getvalue().split() -['first', 'second', 'alpha', 'inserted', 'beta', 'gamma', 'third'] -""" + import sys + sys.stdout.write(reprstr) + """ diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tempita/_tempita.py cython-0.20.1+1~202203241016-9537/Cython/Tempita/_tempita.py --- cython-0.20.1+1~201611251650-6686/Cython/Tempita/_tempita.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tempita/_tempita.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,3 +1,5 @@ +# cython: language_level=3str + """ A small templating language @@ -113,7 +115,7 @@ self.default_namespace['start_braces'] = delimeters[0] self.default_namespace['end_braces'] = delimeters[1] self.delimeters = delimeters - + self._unicode = is_unicode(content) if name is None and stacklevel is not None: try: @@ -144,9 +146,8 @@ def from_filename(cls, filename, namespace=None, encoding=None, default_inherit=None, get_template=get_file_template): - f = open(filename, 'rb') - c = f.read() - f.close() + with open(filename, 'rb') as f: + c = f.read() if encoding: c = c.decode(encoding) return cls(content=c, name=filename, namespace=namespace, @@ -335,7 +336,7 @@ if not isinstance(value, basestring_): value = coerce_text(value) if (is_unicode(value) - and self.default_encoding): + and self.default_encoding): value = value.encode(self.default_encoding) except Exception as e: e.args = (self._add_line_info(e.args[0], pos),) @@ -723,7 +724,7 @@ else: next_chunk = tokens[i + 1] if (not isinstance(next_chunk, basestring_) - or not isinstance(prev, basestring_)): + or not isinstance(prev, basestring_)): continue prev_ok = not prev or trail_whitespace_re.search(prev) if i == 1 and not prev.strip(): @@ -735,7 +736,7 @@ or (i == len(tokens) - 2 and not next_chunk.strip()))): if prev: if ((i == 1 and not prev.strip()) - or prev_ok == 'last'): + or prev_ok == 'last'): tokens[i - 1] = '' else: m = trail_whitespace_re.search(prev) @@ -887,7 +888,7 @@ 'Missing {{endif}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endif'): + and tokens[0][0] == 'endif'): return ('cond', start) + tuple(pieces), tokens[1:] next_chunk, tokens = parse_one_cond(tokens, name, context) pieces.append(next_chunk) @@ -949,7 +950,7 @@ 'No {{endfor}}', position=pos, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endfor'): + and tokens[0][0] == 'endfor'): return ('for', pos, vars, expr, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1009,7 +1010,7 @@ 'Missing {{enddef}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'enddef'): + and tokens[0][0] == 'enddef'): return ('def', start, func_name, sig, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1072,7 +1073,7 @@ raise TemplateError('Invalid signature: (%s)' % sig_text, position=pos, name=name) if (not nest_count and - (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): + (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): default_expr = isolate_expression(sig_text, start_pos, end_pos) defaults[var_name] = default_expr sig_args.append(var_name) @@ -1162,9 +1163,8 @@ template_content = sys.stdin.read() template_name = '' else: - f = open(template_name, 'rb') - template_content = f.read() - f.close() + with open(template_name, 'rb') as f: + template_content = f.read() if options.use_html: TemplateClass = HTMLTemplate else: @@ -1172,9 +1172,8 @@ template = TemplateClass(template_content, name=template_name) result = template.substitute(vars) if options.output: - f = open(options.output, 'wb') - f.write(result) - f.close() + with open(options.output, 'wb') as f: + f.write(result) else: sys.stdout.write(result) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tests/TestCodeWriter.py cython-0.20.1+1~202203241016-9537/Cython/Tests/TestCodeWriter.py --- cython-0.20.1+1~201611251650-6686/Cython/Tests/TestCodeWriter.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tests/TestCodeWriter.py 2022-03-24 10:16:46.000000000 +0000 @@ -4,7 +4,7 @@ # CythonTest uses the CodeWriter heavily, so do some checking by # roundtripping Cython code through the test framework. - # Note that this test is dependant upon the normal Cython parser + # Note that this test is dependent upon the normal Cython parser # to generate the input trees to the CodeWriter. This save *a lot* # of time; better to spend that time writing other tests than perfecting # this one... @@ -19,9 +19,9 @@ def test_print(self): self.t(u""" - print x, y - print x + y ** 2 - print x, y, z, + print(x + y ** 2) + print(x, y, z) + print(x + y, x + y * z, x * (y + z)) """) def test_if(self): @@ -47,6 +47,20 @@ pass """) + def test_cdef(self): + self.t(u""" + cdef f(x, y, z): + pass + cdef public void (x = 34, y = 54, z): + pass + cdef f(int *x, void *y, Value *z): + pass + cdef f(int **x, void **y, Value **z): + pass + cdef inline f(int &x, Value &z): + pass + """) + def test_longness_and_signedness(self): self.t(u"def f(unsigned long long long long long int y):\n pass") @@ -65,18 +79,50 @@ def test_for_loop(self): self.t(u""" for x, y, z in f(g(h(34) * 2) + 23): - print x, y, z + print(x, y, z) + else: + print(43) + """) + self.t(u""" + for abc in (1, 2, 3): + print(x, y, z) else: - print 43 + print(43) + """) + + def test_while_loop(self): + self.t(u""" + while True: + while True: + while True: + continue """) def test_inplace_assignment(self): self.t(u"x += 43") + def test_cascaded_assignment(self): + self.t(u"x = y = z = abc = 43") + def test_attribute(self): self.t(u"a.x") + def test_return_none(self): + self.t(u""" + def f(x, y, z): + return + cdef f(x, y, z): + return + def f(x, y, z): + return None + cdef f(x, y, z): + return None + def f(x, y, z): + return 1234 + cdef f(x, y, z): + return 1234 + """) + if __name__ == "__main__": import unittest unittest.main() - diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tests/TestCythonUtils.py cython-0.20.1+1~202203241016-9537/Cython/Tests/TestCythonUtils.py --- cython-0.20.1+1~201611251650-6686/Cython/Tests/TestCythonUtils.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tests/TestCythonUtils.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,128 @@ +import unittest + +from Cython.Utils import ( + _CACHE_NAME_PATTERN, _build_cache_name, _find_cache_attributes, + build_hex_version, cached_method, clear_method_caches, try_finally_contextmanager) + +METHOD_NAME = "cached_next" +CACHE_NAME = _build_cache_name(METHOD_NAME) +NAMES = CACHE_NAME, METHOD_NAME + +class Cached(object): + @cached_method + def cached_next(self, x): + return next(x) + + +class TestCythonUtils(unittest.TestCase): + def test_build_hex_version(self): + self.assertEqual('0x001D00A1', build_hex_version('0.29a1')) + self.assertEqual('0x001D03C4', build_hex_version('0.29.3rc4')) + self.assertEqual('0x001D00F0', build_hex_version('0.29')) + self.assertEqual('0x040000F0', build_hex_version('4.0')) + + ############################## Cached Methods ############################## + + def test_cache_method_name(self): + method_name = "foo" + cache_name = _build_cache_name(method_name) + match = _CACHE_NAME_PATTERN.match(cache_name) + + self.assertIsNot(match, None) + self.assertEqual(match.group(1), method_name) + + def test_requirements_for_Cached(self): + obj = Cached() + + self.assertFalse(hasattr(obj, CACHE_NAME)) + self.assertTrue(hasattr(obj, METHOD_NAME)) + self.set_of_names_equal(obj, set()) + + def set_of_names_equal(self, obj, value): + self.assertEqual(set(_find_cache_attributes(obj)), value) + + def test_find_cache_attributes(self): + obj = Cached() + method_name = "bar" + cache_name = _build_cache_name(method_name) + + setattr(obj, CACHE_NAME, {}) + setattr(obj, cache_name, {}) + + self.assertFalse(hasattr(obj, method_name)) + self.set_of_names_equal(obj, {NAMES, (cache_name, method_name)}) + + def test_cached_method(self): + obj = Cached() + value = iter(range(3)) # iter for Py2 + cache = {(value,): 0} + + # cache args + self.assertEqual(obj.cached_next(value), 0) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + # use cache + self.assertEqual(obj.cached_next(value), 0) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + def test_clear_method_caches(self): + obj = Cached() + value = iter(range(3)) # iter for Py2 + cache = {(value,): 1} + + obj.cached_next(value) # cache args + + clear_method_caches(obj) + self.set_of_names_equal(obj, set()) + + self.assertEqual(obj.cached_next(value), 1) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + def test_clear_method_caches_with_missing_method(self): + obj = Cached() + method_name = "bar" + cache_name = _build_cache_name(method_name) + names = cache_name, method_name + + setattr(obj, cache_name, object()) + + self.assertFalse(hasattr(obj, method_name)) + self.set_of_names_equal(obj, {names}) + + clear_method_caches(obj) + self.set_of_names_equal(obj, {names}) + + def test_try_finally_contextmanager(self): + states = [] + @try_finally_contextmanager + def gen(*args, **kwargs): + states.append("enter") + yield (args, kwargs) + states.append("exit") + + with gen(1, 2, 3, x=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2, 3), {'x': 4})) + assert states == ["enter", "exit"] + + class MyException(RuntimeError): + pass + + del states[:] + with self.assertRaises(MyException): + with gen(1, 2, y=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2), {'y': 4})) + raise MyException("FAIL INSIDE") + assert states == ["enter", "exit"] + + del states[:] + with self.assertRaises(StopIteration): + with gen(1, 2, y=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2), {'y': 4})) + raise StopIteration("STOP") + assert states == ["enter", "exit"] diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tests/TestJediTyper.py cython-0.20.1+1~202203241016-9537/Cython/Tests/TestJediTyper.py --- cython-0.20.1+1~201611251650-6686/Cython/Tests/TestJediTyper.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tests/TestJediTyper.py 2022-03-24 10:16:46.000000000 +0000 @@ -11,7 +11,7 @@ from tempfile import NamedTemporaryFile from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives -from Cython.Compiler import Main, Symtab, Visitor +from Cython.Compiler import Main, Symtab, Visitor, Options from Cython.TestUtils import TransformTest TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools')) @@ -129,7 +129,7 @@ variables = types.pop((None, (1, 0))) self.assertFalse(types) self.assertEqual({'a': set(['list']), 'b': set(['list']), 'c': set(['list']), 'd': set(['list'])}, variables) - + def test_typing_function_list(self): code = '''\ def func(x): @@ -149,14 +149,14 @@ code = '''\ a = dict() b = {i: i**2 for i in range(10)} - c = a + c = a ''' types = self._test(code) self.assertIn((None, (1, 0)), types) variables = types.pop((None, (1, 0))) self.assertFalse(types) self.assertEqual({'a': set(['dict']), 'b': set(['dict']), 'c': set(['dict'])}, variables) - + def test_typing_function_dict(self): code = '''\ def func(x): @@ -186,7 +186,7 @@ variables = types.pop((None, (1, 0))) self.assertFalse(types) self.assertEqual({'a': set(['set']), 'c': set(['set']), 'd': set(['set']), 'e': set(['set'])}, variables) - + def test_typing_function_set(self): code = '''\ def func(x): @@ -210,8 +210,8 @@ """ def setUp(self): super(TestTypeInjection, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) self.declarations_finder = DeclarationsFinder() diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tests/TestTestUtils.py cython-0.20.1+1~202203241016-9537/Cython/Tests/TestTestUtils.py --- cython-0.20.1+1~201611251650-6686/Cython/Tests/TestTestUtils.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tests/TestTestUtils.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +import os.path +import unittest +import tempfile +import textwrap +import shutil + +from ..TestUtils import write_file, write_newer_file + + +class TestTestUtils(unittest.TestCase): + def setUp(self): + super(TestTestUtils, self).setUp() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if self.temp_dir and os.path.isdir(self.temp_dir): + shutil.rmtree(self.temp_dir) + super(TestTestUtils, self).tearDown() + + def _test_path(self, filename): + return os.path.join(self.temp_dir, filename) + + def _test_write_file(self, content, expected, **kwargs): + file_path = self._test_path("abcfile") + write_file(file_path, content, **kwargs) + assert os.path.isfile(file_path) + + with open(file_path, 'rb') as f: + found = f.read() + assert found == expected, (repr(expected), repr(found)) + + def test_write_file_text(self): + text = u"abcüöä" + self._test_write_file(text, text.encode('utf8')) + + def test_write_file_dedent(self): + text = u""" + A horse is a horse, + of course, of course, + And no one can talk to a horse + of course + """ + self._test_write_file(text, textwrap.dedent(text).encode('utf8'), dedent=True) + + def test_write_file_bytes(self): + self._test_write_file(b"ab\0c", b"ab\0c") + + def test_write_newer_file(self): + file_path_1 = self._test_path("abcfile1.txt") + file_path_2 = self._test_path("abcfile2.txt") + write_file(file_path_1, "abc") + assert os.path.isfile(file_path_1) + write_newer_file(file_path_2, file_path_1, "xyz") + assert os.path.isfile(file_path_2) + assert os.path.getmtime(file_path_2) > os.path.getmtime(file_path_1) + + def test_write_newer_file_same(self): + file_path = self._test_path("abcfile.txt") + write_file(file_path, "abc") + mtime = os.path.getmtime(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.getmtime(file_path) > mtime + + def test_write_newer_file_fresh(self): + file_path = self._test_path("abcfile.txt") + assert not os.path.exists(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.isfile(file_path) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Tests/xmlrunner.py cython-0.20.1+1~202203241016-9537/Cython/Tests/xmlrunner.py --- cython-0.20.1+1~201611251650-6686/Cython/Tests/xmlrunner.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Tests/xmlrunner.py 2022-03-24 10:16:46.000000000 +0000 @@ -27,12 +27,12 @@ def test_choice(self): element = random.choice(self.seq) - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) def test_sample(self): self.assertRaises(ValueError, random.sample, self.seq, 20) for element in random.sample(self.seq, 5): - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) if __name__ == '__main__': unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) @@ -43,7 +43,7 @@ import os import sys import time -from unittest import TestResult, _TextTestResult, TextTestRunner +from unittest import TestResult, TextTestResult, TextTestRunner import xml.dom.minidom try: from StringIO import StringIO @@ -95,7 +95,7 @@ self.err, self.test_method) -class _XMLTestResult(_TextTestResult): +class _XMLTestResult(TextTestResult): """A test result class that can express test results in a XML report. Used by XMLTestRunner. @@ -103,14 +103,13 @@ def __init__(self, stream=sys.stderr, descriptions=1, verbosity=1, elapsed_times=True): "Create a new instance of _XMLTestResult." - _TextTestResult.__init__(self, stream, descriptions, verbosity) + TextTestResult.__init__(self, stream, descriptions, verbosity) self.successes = [] self.callback = None self.elapsed_times = elapsed_times self.output_patched = False - def _prepare_callback(self, test_info, target_list, verbose_str, - short_str): + def _prepare_callback(self, test_info, target_list, verbose_str, short_str): """Append a _TestInfo to the given target list and sets a callback method to be called by stopTest method. """ @@ -125,7 +124,7 @@ self.start_time = self.stop_time = 0 if self.showAll: - self.stream.writeln('(%.3fs) %s' % \ + self.stream.writeln('(%.3fs) %s' % (test_info.get_elapsed_time(), verbose_str)) elif self.dots: self.stream.write(short_str) @@ -159,7 +158,7 @@ def stopTest(self, test): "Called after execute each test method." self._restore_standard_output() - _TextTestResult.stopTest(self, test) + TextTestResult.stopTest(self, test) self.stop_time = time.time() if self.callback and callable(self.callback): @@ -300,8 +299,7 @@ "Generates the XML reports to a given XMLTestRunner object." all_results = self._get_info_by_testcase() - if type(test_runner.output) == str and not \ - os.path.exists(test_runner.output): + if isinstance(test_runner.output, str) and not os.path.exists(test_runner.output): os.makedirs(test_runner.output) for suite, tests in all_results.items(): @@ -321,7 +319,7 @@ xml_content = doc.toprettyxml(indent='\t') if type(test_runner.output) is str: - report_file = open('%s%sTEST-%s.xml' % \ + report_file = open('%s%sTEST-%s.xml' % (test_runner.output, os.sep, suite), 'w') try: report_file.write(xml_content) @@ -348,7 +346,7 @@ """Create the TestResult object which will be used to store information about the executed tests. """ - return _XMLTestResult(self.stream, self.descriptions, \ + return _XMLTestResult(self.stream, self.descriptions, self.verbosity, self.elapsed_times) def run(self, test): diff -Nru cython-0.20.1+1~201611251650-6686/Cython/TestUtils.py cython-0.20.1+1~202203241016-9537/Cython/TestUtils.py --- cython-0.20.1+1~201611251650-6686/Cython/TestUtils.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/TestUtils.py 2022-03-24 10:16:46.000000000 +0000 @@ -2,7 +2,11 @@ import os import unittest +import shlex +import sys import tempfile +import textwrap +from io import open from .Compiler import Errors from .CodeWriter import CodeWriter @@ -47,13 +51,10 @@ class CythonTest(unittest.TestCase): def setUp(self): - self.listing_file = Errors.listing_file - self.echo_file = Errors.echo_file - Errors.listing_file = Errors.echo_file = None + Errors.init_thread() def tearDown(self): - Errors.listing_file = self.listing_file - Errors.echo_file = self.echo_file + Errors.init_thread() def assertLines(self, expected, result): "Checks that the given strings or lists of strings are equal line by line" @@ -177,41 +178,97 @@ if TreePath.find_first(node, path) is not None: Errors.error( node.pos, - "Unexpected path '%s' found in result tree" % path) + "Unexpected path '%s' found in result tree" % path) self.visitchildren(node) return node visit_Node = VisitorTransform.recurse_to_children -def unpack_source_tree(tree_file, dir=None): - if dir is None: - dir = tempfile.mkdtemp() - header = [] - cur_file = None - f = open(tree_file) - try: - lines = f.readlines() - finally: - f.close() - del f +def unpack_source_tree(tree_file, workdir, cython_root): + programs = { + 'PYTHON': [sys.executable], + 'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')], + 'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')] + } + + if workdir is None: + workdir = tempfile.mkdtemp() + header, cur_file = [], None + with open(tree_file, 'rb') as f: + try: + for line in f: + if line[:5] == b'#####': + filename = line.strip().strip(b'#').strip().decode('utf8').replace('/', os.path.sep) + path = os.path.join(workdir, filename) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + if cur_file is not None: + to_close, cur_file = cur_file, None + to_close.close() + cur_file = open(path, 'wb') + elif cur_file is not None: + cur_file.write(line) + elif line.strip() and not line.lstrip().startswith(b'#'): + if line.strip() not in (b'"""', b"'''"): + command = shlex.split(line.decode('utf8')) + if not command: continue + # In Python 3: prog, *args = command + prog, args = command[0], command[1:] + try: + header.append(programs[prog]+args) + except KeyError: + header.append(command) + finally: + if cur_file is not None: + cur_file.close() + return workdir, header + + +def write_file(file_path, content, dedent=False, encoding=None): + r"""Write some content (text or bytes) to the file + at `file_path` without translating `'\n'` into `os.linesep`. + + The default encoding is `'utf-8'`. + """ + if isinstance(content, bytes): + mode = "wb" + + # binary mode doesn't take an encoding and newline arguments + newline = None + default_encoding = None + else: + mode = "w" + + # any "\n" characters written are not translated + # to the system default line separator, os.linesep + newline = "\n" + default_encoding = "utf-8" + + if encoding is None: + encoding = default_encoding + + if dedent: + content = textwrap.dedent(content) + + with open(file_path, mode=mode, encoding=encoding, newline=newline) as f: + f.write(content) + + +def write_newer_file(file_path, newer_than, content, dedent=False, encoding=None): + r""" + Write `content` to the file `file_path` without translating `'\n'` + into `os.linesep` and make sure it is newer than the file `newer_than`. + + The default encoding is `'utf-8'` (same as for `write_file`). + """ + write_file(file_path, content, dedent=dedent, encoding=encoding) + try: - for line in lines: - if line[:5] == '#####': - filename = line.strip().strip('#').strip().replace('/', os.path.sep) - path = os.path.join(dir, filename) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - if cur_file is not None: - f, cur_file = cur_file, None - f.close() - cur_file = open(path, 'w') - elif cur_file is not None: - cur_file.write(line) - elif line.strip() and not line.lstrip().startswith('#'): - if line.strip() not in ('"""', "'''"): - header.append(line) - finally: - if cur_file is not None: - cur_file.close() - return dir, ''.join(header) + other_time = os.path.getmtime(newer_than) + except OSError: + # Support writing a fresh file (which is always newer than a non-existant one) + other_time = None + + while other_time is None or other_time >= os.path.getmtime(file_path): + write_file(file_path, content, dedent=dedent, encoding=encoding) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/arrayarray.h cython-0.20.1+1~202203241016-9537/Cython/Utility/arrayarray.h --- cython-0.20.1+1~201611251650-6686/Cython/Utility/arrayarray.h 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/arrayarray.h 2022-03-24 10:16:46.000000000 +0000 @@ -29,7 +29,7 @@ int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); #if PY_MAJOR_VERSION >= 3 char *formats; -#endif +#endif } arraydescr; @@ -47,6 +47,10 @@ char *as_chars; unsigned long *as_ulongs; long *as_longs; +#if PY_MAJOR_VERSION >= 3 + unsigned long long *as_ulonglongs; + long long *as_longlongs; +#endif short *as_shorts; unsigned short *as_ushorts; Py_UNICODE *as_pyunicodes; @@ -84,7 +88,7 @@ op->ob_descr = descr; op->allocated = size; op->weakreflist = NULL; - op->ob_size = size; + __Pyx_SET_SIZE(op, size); if (size <= 0) { op->data.ob_item = NULL; } @@ -110,9 +114,9 @@ if (items == NULL) { PyErr_NoMemory(); return -1; - } + } self->data.ob_item = (char*) items; - self->ob_size = n; + __Pyx_SET_SIZE(self, n); self->allocated = n; return 0; } @@ -121,12 +125,12 @@ static CYTHON_INLINE int resize_smart(arrayobject *self, Py_ssize_t n) { void *items = (void*) self->data.ob_item; Py_ssize_t newsize; - if (n < self->ob_size) { - self->ob_size = n; + if (n < self->allocated && n*4 > self->allocated) { + __Pyx_SET_SIZE(self, n); return 0; } newsize = n + (n / 2) + 1; - if (newsize <= self->allocated) { /* overflow */ + if (newsize <= n) { /* overflow */ PyErr_NoMemory(); return -1; } @@ -134,9 +138,9 @@ if (items == NULL) { PyErr_NoMemory(); return -1; - } + } self->data.ob_item = (char*) items; - self->ob_size = n; + __Pyx_SET_SIZE(self, n); self->allocated = newsize; return 0; } diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/AsyncGen.c cython-0.20.1+1~202203241016-9537/Cython/Utility/AsyncGen.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/AsyncGen.c 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/AsyncGen.c 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,1284 @@ +// This is copied from genobject.c in CPython 3.6. +// Try to keep it in sync by doing this from time to time: +// sed -e 's|__pyx_||ig' Cython/Utility/AsyncGen.c | diff -udw - cpython/Objects/genobject.c | less + +//////////////////// AsyncGenerator.proto //////////////////// +//@requires: Coroutine.c::Coroutine + +#define __Pyx_AsyncGen_USED +typedef struct { + __pyx_CoroutineObject coro; + PyObject *ag_finalizer; + int ag_hooks_inited; + int ag_closed; + int ag_running_async; +} __pyx_PyAsyncGenObject; + +static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0; +static PyTypeObject *__pyx__PyAsyncGenASendType = 0; +static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0; +static PyTypeObject *__pyx_AsyncGenType = 0; + +#define __Pyx_AsyncGen_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_AsyncGenType) +#define __pyx_PyAsyncGenASend_CheckExact(o) \ + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType) +#define __pyx_PyAsyncGenAThrow_CheckExact(o) \ + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenAThrowType) + +static PyObject *__Pyx_async_gen_anext(PyObject *o); +static CYTHON_INLINE PyObject *__Pyx_async_gen_asend_iternext(PyObject *o); +static PyObject *__Pyx_async_gen_asend_send(PyObject *o, PyObject *arg); +static PyObject *__Pyx_async_gen_asend_close(PyObject *o, PyObject *args); +static PyObject *__Pyx_async_gen_athrow_close(PyObject *o, PyObject *args); + +static PyObject *__Pyx__PyAsyncGenValueWrapperNew(PyObject *val); + + +static __pyx_CoroutineObject *__Pyx_AsyncGen_New( + __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { + __pyx_PyAsyncGenObject *gen = PyObject_GC_New(__pyx_PyAsyncGenObject, __pyx_AsyncGenType); + if (unlikely(!gen)) + return NULL; + gen->ag_finalizer = NULL; + gen->ag_closed = 0; + gen->ag_hooks_inited = 0; + gen->ag_running_async = 0; + return __Pyx__Coroutine_NewInit((__pyx_CoroutineObject*)gen, body, code, closure, name, qualname, module_name); +} + +static int __pyx_AsyncGen_init(PyObject *module); +static void __Pyx_PyAsyncGen_Fini(void); + +//////////////////// AsyncGenerator.cleanup //////////////////// + +__Pyx_PyAsyncGen_Fini(); + +//////////////////// AsyncGeneratorInitFinalizer //////////////////// + +// this is separated out because it needs more adaptation + +#if PY_VERSION_HEX < 0x030600B0 +static int __Pyx_async_gen_init_hooks(__pyx_PyAsyncGenObject *o) { +#if 0 + // TODO: implement finalizer support in older Python versions + PyThreadState *tstate; + PyObject *finalizer; + PyObject *firstiter; +#endif + + if (likely(o->ag_hooks_inited)) { + return 0; + } + + o->ag_hooks_inited = 1; + +#if 0 + tstate = __Pyx_PyThreadState_Current; + + finalizer = tstate->async_gen_finalizer; + if (finalizer) { + Py_INCREF(finalizer); + o->ag_finalizer = finalizer; + } + + firstiter = tstate->async_gen_firstiter; + if (firstiter) { + PyObject *res; + + Py_INCREF(firstiter); + res = __Pyx_PyObject_CallOneArg(firstiter, (PyObject*)o); + Py_DECREF(firstiter); + if (res == NULL) { + return 1; + } + Py_DECREF(res); + } +#endif + + return 0; +} +#endif + + +//////////////////// AsyncGenerator //////////////////// +//@requires: AsyncGeneratorInitFinalizer +//@requires: Coroutine.c::Coroutine +//@requires: Coroutine.c::ReturnWithStopIteration +//@requires: ObjectHandling.c::PyObjectCall2Args +//@requires: ObjectHandling.c::PyObject_GenericGetAttrNoDict + +PyDoc_STRVAR(__Pyx_async_gen_send_doc, +"send(arg) -> send 'arg' into generator,\n\ +return next yielded value or raise StopIteration."); + +PyDoc_STRVAR(__Pyx_async_gen_close_doc, +"close() -> raise GeneratorExit inside generator."); + +PyDoc_STRVAR(__Pyx_async_gen_throw_doc, +"throw(typ[,val[,tb]]) -> raise exception in generator,\n\ +return next yielded value or raise StopIteration."); + +PyDoc_STRVAR(__Pyx_async_gen_await_doc, +"__await__() -> return a representation that can be passed into the 'await' expression."); + +// COPY STARTS HERE: + +static PyObject *__Pyx_async_gen_asend_new(__pyx_PyAsyncGenObject *, PyObject *); +static PyObject *__Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *, PyObject *); + +static const char *__Pyx_NON_INIT_CORO_MSG = "can't send non-None value to a just-started coroutine"; +static const char *__Pyx_ASYNC_GEN_IGNORED_EXIT_MSG = "async generator ignored GeneratorExit"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG = "cannot reuse already awaited __anext__()/asend()"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG = "cannot reuse already awaited aclose()/athrow()"; + +typedef enum { + __PYX_AWAITABLE_STATE_INIT, /* new awaitable, has not yet been iterated */ + __PYX_AWAITABLE_STATE_ITER, /* being iterated */ + __PYX_AWAITABLE_STATE_CLOSED, /* closed */ +} __pyx_AwaitableState; + +typedef struct { + PyObject_HEAD + __pyx_PyAsyncGenObject *ags_gen; + + /* Can be NULL, when in the __anext__() mode (equivalent of "asend(None)") */ + PyObject *ags_sendval; + + __pyx_AwaitableState ags_state; +} __pyx_PyAsyncGenASend; + + +typedef struct { + PyObject_HEAD + __pyx_PyAsyncGenObject *agt_gen; + + /* Can be NULL, when in the "aclose()" mode (equivalent of "athrow(GeneratorExit)") */ + PyObject *agt_args; + + __pyx_AwaitableState agt_state; +} __pyx_PyAsyncGenAThrow; + + +typedef struct { + PyObject_HEAD + PyObject *agw_val; +} __pyx__PyAsyncGenWrappedValue; + + +#ifndef _PyAsyncGen_MAXFREELIST +#define _PyAsyncGen_MAXFREELIST 80 +#endif + +// Freelists boost performance 6-10%; they also reduce memory +// fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend +// are short-living objects that are instantiated for every +// __anext__ call. + +static __pyx__PyAsyncGenWrappedValue *__Pyx_ag_value_freelist[_PyAsyncGen_MAXFREELIST]; +static int __Pyx_ag_value_freelist_free = 0; + +static __pyx_PyAsyncGenASend *__Pyx_ag_asend_freelist[_PyAsyncGen_MAXFREELIST]; +static int __Pyx_ag_asend_freelist_free = 0; + +#define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \ + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenWrappedValueType) + + +static int +__Pyx_async_gen_traverse(__pyx_PyAsyncGenObject *gen, visitproc visit, void *arg) +{ + Py_VISIT(gen->ag_finalizer); + return __Pyx_Coroutine_traverse((__pyx_CoroutineObject*)gen, visit, arg); +} + + +static PyObject * +__Pyx_async_gen_repr(__pyx_CoroutineObject *o) +{ + // avoid NULL pointer dereference for qualname during garbage collection + return PyUnicode_FromFormat("", + o->gi_qualname ? o->gi_qualname : Py_None, o); +} + + +#if PY_VERSION_HEX >= 0x030600B0 +static int +__Pyx_async_gen_init_hooks(__pyx_PyAsyncGenObject *o) +{ + PyThreadState *tstate; + PyObject *finalizer; + PyObject *firstiter; + + if (o->ag_hooks_inited) { + return 0; + } + + o->ag_hooks_inited = 1; + + tstate = __Pyx_PyThreadState_Current; + + finalizer = tstate->async_gen_finalizer; + if (finalizer) { + Py_INCREF(finalizer); + o->ag_finalizer = finalizer; + } + + firstiter = tstate->async_gen_firstiter; + if (firstiter) { + PyObject *res; +#if CYTHON_UNPACK_METHODS + PyObject *self; +#endif + + Py_INCREF(firstiter); + // at least asyncio stores methods here => optimise the call +#if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(firstiter)) && likely((self = PyMethod_GET_SELF(firstiter)) != NULL)) { + PyObject *function = PyMethod_GET_FUNCTION(firstiter); + res = __Pyx_PyObject_Call2Args(function, self, (PyObject*)o); + } else +#endif + res = __Pyx_PyObject_CallOneArg(firstiter, (PyObject*)o); + + Py_DECREF(firstiter); + if (unlikely(res == NULL)) { + return 1; + } + Py_DECREF(res); + } + + return 0; +} +#endif + + +static PyObject * +__Pyx_async_gen_anext(PyObject *g) +{ + __pyx_PyAsyncGenObject *o = (__pyx_PyAsyncGenObject*) g; + if (unlikely(__Pyx_async_gen_init_hooks(o))) { + return NULL; + } + return __Pyx_async_gen_asend_new(o, NULL); +} + +static PyObject * +__Pyx_async_gen_anext_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + return __Pyx_async_gen_anext(g); +} + + +static PyObject * +__Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) +{ + if (unlikely(__Pyx_async_gen_init_hooks(o))) { + return NULL; + } + return __Pyx_async_gen_asend_new(o, arg); +} + + +static PyObject * +__Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, PyObject *arg) +{ + CYTHON_UNUSED_VAR(arg); + if (unlikely(__Pyx_async_gen_init_hooks(o))) { + return NULL; + } + return __Pyx_async_gen_athrow_new(o, NULL); +} + + +static PyObject * +__Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) +{ + if (unlikely(__Pyx_async_gen_init_hooks(o))) { + return NULL; + } + return __Pyx_async_gen_athrow_new(o, args); +} + + +static PyObject * +__Pyx_async_gen_self_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + return __Pyx_NewRef(g); +} + + +static PyGetSetDef __Pyx_async_gen_getsetlist[] = { + {(char*) "__name__", (getter)__Pyx_Coroutine_get_name, (setter)__Pyx_Coroutine_set_name, + (char*) PyDoc_STR("name of the async generator"), 0}, + {(char*) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, + (char*) PyDoc_STR("qualified name of the async generator"), 0}, + //REMOVED: {(char*) "ag_await", (getter)coro_get_cr_await, NULL, + //REMOVED: (char*) PyDoc_STR("object being awaited on, or None")}, + {0, 0, 0, 0, 0} /* Sentinel */ +}; + +static PyMemberDef __Pyx_async_gen_memberlist[] = { + //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, + {(char*) "ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, + //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, + //ADDED: "ag_await" + {(char*) "ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, + (char*) PyDoc_STR("object being awaited on, or None")}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif + {0, 0, 0, 0, 0} /* Sentinel */ +}; + +PyDoc_STRVAR(__Pyx_async_aclose_doc, +"aclose() -> raise GeneratorExit inside generator."); + +PyDoc_STRVAR(__Pyx_async_asend_doc, +"asend(v) -> send 'v' in generator."); + +PyDoc_STRVAR(__Pyx_async_athrow_doc, +"athrow(typ[,val[,tb]]) -> raise exception in generator."); + +PyDoc_STRVAR(__Pyx_async_aiter_doc, +"__aiter__(v) -> return an asynchronous iterator."); + +PyDoc_STRVAR(__Pyx_async_anext_doc, +"__anext__(v) -> continue asynchronous iteration and return the next element."); + +static PyMethodDef __Pyx_async_gen_methods[] = { + {"asend", (PyCFunction)__Pyx_async_gen_asend, METH_O, __Pyx_async_asend_doc}, + {"athrow",(PyCFunction)__Pyx_async_gen_athrow, METH_VARARGS, __Pyx_async_athrow_doc}, + {"aclose", (PyCFunction)__Pyx_async_gen_aclose, METH_NOARGS, __Pyx_async_aclose_doc}, + {"__aiter__", (PyCFunction)__Pyx_async_gen_self_method, METH_NOARGS, __Pyx_async_aiter_doc}, + {"__anext__", (PyCFunction)__Pyx_async_gen_anext_method, METH_NOARGS, __Pyx_async_anext_doc}, + {0, 0, 0, 0} /* Sentinel */ +}; + + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_AsyncGenType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_aiter, (void *)PyObject_SelfIter}, + {Py_am_anext, (void *)__Pyx_async_gen_anext}, + {Py_tp_repr, (void *)__Pyx_async_gen_repr}, + {Py_tp_traverse, (void *)__Pyx_async_gen_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_methods}, + {Py_tp_members, (void *)__Pyx_async_gen_memberlist}, + {Py_tp_getset, (void *)__Pyx_async_gen_getsetlist}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_AsyncGenType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator", + sizeof(__pyx_PyAsyncGenObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_AsyncGenType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + +#if CYTHON_USE_ASYNC_SLOTS +static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_as_async = { + 0, /* am_await */ + PyObject_SelfIter, /* am_aiter */ + (unaryfunc)__Pyx_async_gen_anext, /* am_anext */ +#if PY_VERSION_HEX >= 0x030A00A3 + 0, /*am_send*/ +#endif +}; +#endif + +static PyTypeObject __pyx_AsyncGenType_type = { + PyVarObject_HEAD_INIT(0, 0) + "async_generator", /* tp_name */ + sizeof(__pyx_PyAsyncGenObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)__Pyx_Coroutine_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#if CYTHON_USE_ASYNC_SLOTS + &__Pyx_async_gen_as_async, /* tp_as_async */ +#else + 0, /*tp_reserved*/ +#endif + (reprfunc)__Pyx_async_gen_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_HAVE_FINALIZE, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)__Pyx_async_gen_traverse, /* tp_traverse */ + 0, /* tp_clear */ +#if CYTHON_USE_ASYNC_SLOTS && CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 + // in order to (mis-)use tp_reserved above, we must also implement tp_richcompare + __Pyx_Coroutine_compare, /*tp_richcompare*/ +#else + 0, /*tp_richcompare*/ +#endif + offsetof(__pyx_CoroutineObject, gi_weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + __Pyx_async_gen_methods, /* tp_methods */ + __Pyx_async_gen_memberlist, /* tp_members */ + __Pyx_async_gen_getsetlist, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ +#if CYTHON_USE_TP_FINALIZE + 0, /*tp_del*/ +#else + __Pyx_Coroutine_del, /*tp_del*/ +#endif + 0, /* tp_version_tag */ +#if CYTHON_USE_TP_FINALIZE + __Pyx_Coroutine_del, /* tp_finalize */ +#elif PY_VERSION_HEX >= 0x030400a1 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif +}; +#endif /* CYTHON_USE_TYPE_SPECS */ + + +static int +__Pyx_PyAsyncGen_ClearFreeLists(void) +{ + int ret = __Pyx_ag_value_freelist_free + __Pyx_ag_asend_freelist_free; + + while (__Pyx_ag_value_freelist_free) { + __pyx__PyAsyncGenWrappedValue *o; + o = __Pyx_ag_value_freelist[--__Pyx_ag_value_freelist_free]; + assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); + __Pyx_PyHeapTypeObject_GC_Del(o); + } + + while (__Pyx_ag_asend_freelist_free) { + __pyx_PyAsyncGenASend *o; + o = __Pyx_ag_asend_freelist[--__Pyx_ag_asend_freelist_free]; + assert(__Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType)); + __Pyx_PyHeapTypeObject_GC_Del(o); + } + + return ret; +} + +static void +__Pyx_PyAsyncGen_Fini(void) +{ + __Pyx_PyAsyncGen_ClearFreeLists(); +} + + +static PyObject * +__Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) +{ + if (result == NULL) { + PyObject *exc_type = PyErr_Occurred(); + if (!exc_type) { + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); + gen->ag_closed = 1; + } else if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + gen->ag_closed = 1; + } + + gen->ag_running_async = 0; + return NULL; + } + + if (__pyx__PyAsyncGenWrappedValue_CheckExact(result)) { + /* async yield */ + __Pyx_ReturnWithStopIteration(((__pyx__PyAsyncGenWrappedValue*)result)->agw_val); + Py_DECREF(result); + gen->ag_running_async = 0; + return NULL; + } + + return result; +} + + +/* ---------- Async Generator ASend Awaitable ------------ */ + + +static void +__Pyx_async_gen_asend_dealloc(__pyx_PyAsyncGenASend *o) +{ + PyObject_GC_UnTrack((PyObject *)o); + Py_CLEAR(o->ags_gen); + Py_CLEAR(o->ags_sendval); + if (likely(__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST)) { + assert(__pyx_PyAsyncGenASend_CheckExact(o)); + __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free++] = o; + } else { + __Pyx_PyHeapTypeObject_GC_Del(o); + } +} + +static int +__Pyx_async_gen_asend_traverse(__pyx_PyAsyncGenASend *o, visitproc visit, void *arg) +{ + Py_VISIT(o->ags_gen); + Py_VISIT(o->ags_sendval); + return 0; +} + + +static PyObject * +__Pyx_async_gen_asend_send(PyObject *g, PyObject *arg) +{ + __pyx_PyAsyncGenASend *o = (__pyx_PyAsyncGenASend*) g; + PyObject *result; + + if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); + return NULL; + } + + if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) { + if (unlikely(o->ags_gen->ag_running_async)) { + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + if (arg == NULL || arg == Py_None) { + arg = o->ags_sendval ? o->ags_sendval : Py_None; + } + o->ags_state = __PYX_AWAITABLE_STATE_ITER; + } + + o->ags_gen->ag_running_async = 1; + result = __Pyx_Coroutine_Send((PyObject*)o->ags_gen, arg); + result = __Pyx_async_gen_unwrap_value(o->ags_gen, result); + + if (result == NULL) { + o->ags_state = __PYX_AWAITABLE_STATE_CLOSED; + } + + return result; +} + + +static CYTHON_INLINE PyObject * +__Pyx_async_gen_asend_iternext(PyObject *o) +{ + return __Pyx_async_gen_asend_send(o, Py_None); +} + + +static PyObject * +__Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) +{ + PyObject *result; + + if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); + return NULL; + } + + result = __Pyx_Coroutine_Throw((PyObject*)o->ags_gen, args); + result = __Pyx_async_gen_unwrap_value(o->ags_gen, result); + + if (result == NULL) { + o->ags_state = __PYX_AWAITABLE_STATE_CLOSED; + } + + return result; +} + + +static PyObject * +__Pyx_async_gen_asend_close(PyObject *g, PyObject *args) +{ + __pyx_PyAsyncGenASend *o = (__pyx_PyAsyncGenASend*) g; + CYTHON_UNUSED_VAR(args); + o->ags_state = __PYX_AWAITABLE_STATE_CLOSED; + Py_RETURN_NONE; +} + + +static PyMethodDef __Pyx_async_gen_asend_methods[] = { + {"send", (PyCFunction)__Pyx_async_gen_asend_send, METH_O, __Pyx_async_gen_send_doc}, + {"throw", (PyCFunction)__Pyx_async_gen_asend_throw, METH_VARARGS, __Pyx_async_gen_throw_doc}, + {"close", (PyCFunction)__Pyx_async_gen_asend_close, METH_NOARGS, __Pyx_async_gen_close_doc}, + {"__await__", (PyCFunction)__Pyx_async_gen_self_method, METH_NOARGS, __Pyx_async_gen_await_doc}, + {0, 0, 0, 0} /* Sentinel */ +}; + + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenASendType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_asend_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_asend_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_asend_methods}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_asend_iternext}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenASendType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_asend", + sizeof(__pyx_PyAsyncGenASend), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenASendType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + +#if CYTHON_USE_ASYNC_SLOTS +static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ +#if PY_VERSION_HEX >= 0x030A00A3 + 0, /*am_send*/ +#endif +}; +#endif + +static PyTypeObject __pyx__PyAsyncGenASendType_type = { + PyVarObject_HEAD_INIT(0, 0) + "async_generator_asend", /* tp_name */ + sizeof(__pyx_PyAsyncGenASend), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)__Pyx_async_gen_asend_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#if CYTHON_USE_ASYNC_SLOTS + &__Pyx_async_gen_asend_as_async, /* tp_as_async */ +#else + 0, /*tp_reserved*/ +#endif + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)__Pyx_async_gen_asend_traverse, /* tp_traverse */ + 0, /* tp_clear */ +#if CYTHON_USE_ASYNC_SLOTS && CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 + // in order to (mis-)use tp_reserved above, we must also implement tp_richcompare + __Pyx_Coroutine_compare, /*tp_richcompare*/ +#else + 0, /*tp_richcompare*/ +#endif + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)__Pyx_async_gen_asend_iternext, /* tp_iternext */ + __Pyx_async_gen_asend_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x030400a1 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif +}; +#endif /* CYTHON_USE_TYPE_SPECS */ + + +static PyObject * +__Pyx_async_gen_asend_new(__pyx_PyAsyncGenObject *gen, PyObject *sendval) +{ + __pyx_PyAsyncGenASend *o; + if (likely(__Pyx_ag_asend_freelist_free)) { + __Pyx_ag_asend_freelist_free--; + o = __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free]; + _Py_NewReference((PyObject *)o); + } else { + o = PyObject_GC_New(__pyx_PyAsyncGenASend, __pyx__PyAsyncGenASendType); + if (unlikely(o == NULL)) { + return NULL; + } + } + + Py_INCREF(gen); + o->ags_gen = gen; + + Py_XINCREF(sendval); + o->ags_sendval = sendval; + + o->ags_state = __PYX_AWAITABLE_STATE_INIT; + + PyObject_GC_Track((PyObject*)o); + return (PyObject*)o; +} + + +/* ---------- Async Generator Value Wrapper ------------ */ + + +static void +__Pyx_async_gen_wrapped_val_dealloc(__pyx__PyAsyncGenWrappedValue *o) +{ + PyObject_GC_UnTrack((PyObject *)o); + Py_CLEAR(o->agw_val); + if (likely(__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST)) { + assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); + __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free++] = o; + } else { + __Pyx_PyHeapTypeObject_GC_Del(o); + } +} + + +static int +__Pyx_async_gen_wrapped_val_traverse(__pyx__PyAsyncGenWrappedValue *o, + visitproc visit, void *arg) +{ + Py_VISIT(o->agw_val); + return 0; +} + + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenWrappedValueType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_wrapped_val_dealloc}, + {Py_tp_traverse, (void *)__Pyx_async_gen_wrapped_val_traverse}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenWrappedValueType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_wrapped_value", + sizeof(__pyx__PyAsyncGenWrappedValue), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenWrappedValueType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + +static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { + PyVarObject_HEAD_INIT(0, 0) + "async_generator_wrapped_value", /* tp_name */ + sizeof(__pyx__PyAsyncGenWrappedValue), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)__Pyx_async_gen_wrapped_val_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)__Pyx_async_gen_wrapped_val_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x030400a1 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif +}; +#endif /* CYTHON_USE_TYPE_SPECS */ + + +static PyObject * +__Pyx__PyAsyncGenValueWrapperNew(PyObject *val) +{ + // NOTE: steals a reference to val ! + __pyx__PyAsyncGenWrappedValue *o; + assert(val); + + if (likely(__Pyx_ag_value_freelist_free)) { + __Pyx_ag_value_freelist_free--; + o = __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free]; + assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); + _Py_NewReference((PyObject*)o); + } else { + o = PyObject_GC_New(__pyx__PyAsyncGenWrappedValue, __pyx__PyAsyncGenWrappedValueType); + if (unlikely(!o)) { + Py_DECREF(val); + return NULL; + } + } + o->agw_val = val; + // no Py_INCREF(val) - steals reference! + PyObject_GC_Track((PyObject*)o); + return (PyObject*)o; +} + + +/* ---------- Async Generator AThrow awaitable ------------ */ + + +static void +__Pyx_async_gen_athrow_dealloc(__pyx_PyAsyncGenAThrow *o) +{ + PyObject_GC_UnTrack((PyObject *)o); + Py_CLEAR(o->agt_gen); + Py_CLEAR(o->agt_args); + __Pyx_PyHeapTypeObject_GC_Del(o); +} + + +static int +__Pyx_async_gen_athrow_traverse(__pyx_PyAsyncGenAThrow *o, visitproc visit, void *arg) +{ + Py_VISIT(o->agt_gen); + Py_VISIT(o->agt_args); + return 0; +} + + +static PyObject * +__Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) +{ + __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*)o->agt_gen; + PyObject *retval, *exc_type; + + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); + return NULL; + } + + if (unlikely(gen->resume_label == -1)) { + // already run past the end + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + + if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { + if (unlikely(o->agt_gen->ag_running_async)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + if (unlikely(o->agt_gen->ag_closed)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); + return NULL; + } + + if (unlikely(arg != Py_None)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); + return NULL; + } + + o->agt_state = __PYX_AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + + if (o->agt_args == NULL) { + /* aclose() mode */ + o->agt_gen->ag_closed = 1; + + retval = __Pyx__Coroutine_Throw((PyObject*)gen, + /* Do not close generator when PyExc_GeneratorExit is passed */ + PyExc_GeneratorExit, NULL, NULL, NULL, 0); + + if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + Py_DECREF(retval); + goto yield_close; + } + } else { + PyObject *typ; + PyObject *tb = NULL; + PyObject *val = NULL; + + if (unlikely(!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, &typ, &val, &tb))) { + return NULL; + } + + retval = __Pyx__Coroutine_Throw((PyObject*)gen, + /* Do not close generator when PyExc_GeneratorExit is passed */ + typ, val, tb, o->agt_args, 0); + retval = __Pyx_async_gen_unwrap_value(o->agt_gen, retval); + } + if (retval == NULL) { + goto check_error; + } + return retval; + } + + assert (o->agt_state == __PYX_AWAITABLE_STATE_ITER); + + retval = __Pyx_Coroutine_Send((PyObject *)gen, arg); + if (o->agt_args) { + return __Pyx_async_gen_unwrap_value(o->agt_gen, retval); + } else { + /* aclose() mode */ + if (retval) { + if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { + Py_DECREF(retval); + goto yield_close; + } + else { + return retval; + } + } + else { + goto check_error; + } + } + +yield_close: + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); + return NULL; + +check_error: + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + if (o->agt_args == NULL) { + // when aclose() is called we don't want to propagate + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. + PyErr_Clear(); + PyErr_SetNone(PyExc_StopIteration); + } + } + return NULL; +} + + +static PyObject * +__Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) +{ + PyObject *retval; + + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); + return NULL; + } + + retval = __Pyx_Coroutine_Throw((PyObject*)o->agt_gen, args); + if (o->agt_args) { + return __Pyx_async_gen_unwrap_value(o->agt_gen, retval); + } else { + // aclose() mode + PyObject *exc_type; + if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + Py_DECREF(retval); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); + return NULL; + } + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + // when aclose() is called we don't want to propagate + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. + PyErr_Clear(); + PyErr_SetNone(PyExc_StopIteration); + } + return retval; + } +} + + +static PyObject * +__Pyx_async_gen_athrow_iternext(__pyx_PyAsyncGenAThrow *o) +{ + return __Pyx_async_gen_athrow_send(o, Py_None); +} + + +static PyObject * +__Pyx_async_gen_athrow_close(PyObject *g, PyObject *args) +{ + __pyx_PyAsyncGenAThrow *o = (__pyx_PyAsyncGenAThrow*) g; + CYTHON_UNUSED_VAR(args); + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + Py_RETURN_NONE; +} + + +static PyMethodDef __Pyx_async_gen_athrow_methods[] = { + {"send", (PyCFunction)__Pyx_async_gen_athrow_send, METH_O, __Pyx_async_gen_send_doc}, + {"throw", (PyCFunction)__Pyx_async_gen_athrow_throw, METH_VARARGS, __Pyx_async_gen_throw_doc}, + {"close", (PyCFunction)__Pyx_async_gen_athrow_close, METH_NOARGS, __Pyx_async_gen_close_doc}, + {"__await__", (PyCFunction)__Pyx_async_gen_self_method, METH_NOARGS, __Pyx_async_gen_await_doc}, + {0, 0, 0, 0} /* Sentinel */ +}; + + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenAThrowType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_athrow_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_athrow_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_athrow_iternext}, + {Py_tp_methods, (void *)__Pyx_async_gen_athrow_methods}, + {Py_tp_getattro, (void *)__Pyx_PyObject_GenericGetAttrNoDict}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenAThrowType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_athrow", + sizeof(__pyx_PyAsyncGenAThrow), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenAThrowType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + +#if CYTHON_USE_ASYNC_SLOTS +static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = { + PyObject_SelfIter, /* am_await */ + 0, /* am_aiter */ + 0, /* am_anext */ +#if PY_VERSION_HEX >= 0x030A00A3 + 0, /*am_send*/ +#endif +}; +#endif + +static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { + PyVarObject_HEAD_INIT(0, 0) + "async_generator_athrow", /* tp_name */ + sizeof(__pyx_PyAsyncGenAThrow), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)__Pyx_async_gen_athrow_dealloc, /* tp_dealloc */ + 0, /* tp_vectorcall_offset */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#if CYTHON_USE_ASYNC_SLOTS + &__Pyx_async_gen_athrow_as_async, /* tp_as_async */ +#else + 0, /*tp_reserved*/ +#endif + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)__Pyx_async_gen_athrow_traverse, /* tp_traverse */ + 0, /* tp_clear */ +#if CYTHON_USE_ASYNC_SLOTS && CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 + // in order to (mis-)use tp_reserved above, we must also implement tp_richcompare + __Pyx_Coroutine_compare, /*tp_richcompare*/ +#else + 0, /*tp_richcompare*/ +#endif + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)__Pyx_async_gen_athrow_iternext, /* tp_iternext */ + __Pyx_async_gen_athrow_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x030400a1 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif +}; +#endif /* CYTHON_USE_TYPE_SPECS */ + + +static PyObject * +__Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) +{ + __pyx_PyAsyncGenAThrow *o; + o = PyObject_GC_New(__pyx_PyAsyncGenAThrow, __pyx__PyAsyncGenAThrowType); + if (unlikely(o == NULL)) { + return NULL; + } + o->agt_gen = gen; + o->agt_args = args; + o->agt_state = __PYX_AWAITABLE_STATE_INIT; + Py_INCREF(gen); + Py_XINCREF(args); + PyObject_GC_Track((PyObject*)o); + return (PyObject*)o; +} + + +/* ---------- global type sharing ------------ */ + +static int __pyx_AsyncGen_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_AsyncGenType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_AsyncGenType_spec, NULL); +#else + (void) module; + // on Windows, C-API functions can't be used in slots statically + __pyx_AsyncGenType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + __pyx_AsyncGenType = __Pyx_FetchCommonType(&__pyx_AsyncGenType_type); +#endif + if (unlikely(!__pyx_AsyncGenType)) + return -1; + +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenAThrowType_spec, NULL); +#else + __pyx__PyAsyncGenAThrowType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenAThrowType_type); +#endif + if (unlikely(!__pyx__PyAsyncGenAThrowType)) + return -1; + +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenWrappedValueType_spec, NULL); +#else + __pyx__PyAsyncGenWrappedValueType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenWrappedValueType_type); +#endif + if (unlikely(!__pyx__PyAsyncGenWrappedValueType)) + return -1; + +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenASendType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenASendType_spec, NULL); +#else + __pyx__PyAsyncGenASendType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + __pyx__PyAsyncGenASendType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenASendType_type); +#endif + if (unlikely(!__pyx__PyAsyncGenASendType)) + return -1; + + return 0; +} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Buffer.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Buffer.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Buffer.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Buffer.c 2022-03-24 10:16:46.000000000 +0000 @@ -52,6 +52,7 @@ } /////////////// BufferFormatStructs.proto /////////////// +//@proto_block: utility_code_proto_before_types #define IS_UNSIGNED(type) (((type) -1) > 0) @@ -95,7 +96,9 @@ char is_valid_array; } __Pyx_BufFmt_Context; + /////////////// GetAndReleaseBuffer.proto /////////////// + #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags); static void __Pyx_ReleaseBuffer(Py_buffer *view); @@ -105,17 +108,23 @@ #endif /////////////// GetAndReleaseBuffer /////////////// + #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + __Pyx_TypeName obj_type_name; if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags); {{for type_ptr, getbuffer, releasebuffer in types}} {{if getbuffer}} - if (PyObject_TypeCheck(obj, {{type_ptr}})) return {{getbuffer}}(obj, view, flags); + if (__Pyx_TypeCheck(obj, {{type_ptr}})) return {{getbuffer}}(obj, view, flags); {{endif}} {{endfor}} - PyErr_Format(PyExc_TypeError, "'%.200s' does not have the buffer interface", Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' does not have the buffer interface", + obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return -1; } @@ -128,49 +137,111 @@ return; } + if ((0)) {} {{for type_ptr, getbuffer, releasebuffer in types}} {{if releasebuffer}} - if (PyObject_TypeCheck(obj, {{type_ptr}})) { {{releasebuffer}}(obj, view); return; } + else if (__Pyx_TypeCheck(obj, {{type_ptr}})) {{releasebuffer}}(obj, view); {{endif}} {{endfor}} - Py_DECREF(obj); view->obj = NULL; + Py_DECREF(obj); } #endif /* PY_MAJOR_VERSION < 3 */ -/////////////// BufferFormatCheck.proto /////////////// -{{# - Buffer format string checking +/////////////// BufferGetAndValidate.proto /////////////// - Buffer type checking. Utility code for checking that acquired - buffers match our assumptions. We only need to check ndim and - the format string; the access mode/flags is checked by the - exporter. See: +#define __Pyx_GetBufferAndValidate(buf, obj, dtype, flags, nd, cast, stack) \ + ((obj == Py_None || obj == NULL) ? \ + (__Pyx_ZeroBuffer(buf), 0) : \ + __Pyx__GetBufferAndValidate(buf, obj, dtype, flags, nd, cast, stack)) - http://docs.python.org/3/library/struct.html - http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax +static int __Pyx__GetBufferAndValidate(Py_buffer* buf, PyObject* obj, + __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack); +static void __Pyx_ZeroBuffer(Py_buffer* buf); +static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info);/*proto*/ - The alignment code is copied from _struct.c in Python. -}} +static Py_ssize_t __Pyx_minusones[] = { {{ ", ".join(["-1"] * max_dims) }} }; +static Py_ssize_t __Pyx_zeros[] = { {{ ", ".join(["0"] * max_dims) }} }; + + +/////////////// BufferGetAndValidate /////////////// +//@requires: BufferFormatCheck + +static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) { + if (unlikely(info->buf == NULL)) return; + if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL; + __Pyx_ReleaseBuffer(info); +} + +static void __Pyx_ZeroBuffer(Py_buffer* buf) { + buf->buf = NULL; + buf->obj = NULL; + buf->strides = __Pyx_zeros; + buf->shape = __Pyx_zeros; + buf->suboffsets = __Pyx_minusones; +} + +static int __Pyx__GetBufferAndValidate( + Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, + int nd, int cast, __Pyx_BufFmt_StackElem* stack) +{ + buf->buf = NULL; + if (unlikely(__Pyx_GetBuffer(obj, buf, flags) == -1)) { + __Pyx_ZeroBuffer(buf); + return -1; + } + // From this point on, we have acquired the buffer and must release it on errors. + if (unlikely(buf->ndim != nd)) { + PyErr_Format(PyExc_ValueError, + "Buffer has wrong number of dimensions (expected %d, got %d)", + nd, buf->ndim); + goto fail; + } + if (!cast) { + __Pyx_BufFmt_Context ctx; + __Pyx_BufFmt_Init(&ctx, stack, dtype); + if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail; + } + if (unlikely((size_t)buf->itemsize != dtype->size)) { + PyErr_Format(PyExc_ValueError, + "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)", + buf->itemsize, (buf->itemsize > 1) ? "s" : "", + dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : ""); + goto fail; + } + if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones; + return 0; +fail:; + __Pyx_SafeReleaseBuffer(buf); + return -1; +} + + +/////////////// BufferFormatCheck.proto /////////////// + +// Buffer format string checking +// +// Buffer type checking. Utility code for checking that acquired +// buffers match our assumptions. We only need to check ndim and +// the format string; the access mode/flags is checked by the +// exporter. See: +// +// https://docs.python.org/3/library/struct.html +// https://www.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax +// +// The alignment code is copied from _struct.c in Python. -static CYTHON_INLINE int __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj, - __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack); -static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info); static const char* __Pyx_BufFmt_CheckString(__Pyx_BufFmt_Context* ctx, const char* ts); static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, __Pyx_BufFmt_StackElem* stack, - __Pyx_TypeInfo* type); // PROTO - + __Pyx_TypeInfo* type); /*proto*/ /////////////// BufferFormatCheck /////////////// -static CYTHON_INLINE int __Pyx_IsLittleEndian(void) { - unsigned int n = 1; - return *(unsigned char*)(&n) != 0; -} - +//@requires: ModuleSetupCode.c::IsLittleEndian +//@requires: BufferFormatStructs static void __Pyx_BufFmt_Init(__Pyx_BufFmt_Context* ctx, __Pyx_BufFmt_StackElem* stack, @@ -207,7 +278,7 @@ return -1; } else { count = *t++ - '0'; - while (*t >= '0' && *t < '9') { + while (*t >= '0' && *t <= '9') { count *= 10; count += *t++ - '0'; } @@ -232,6 +303,7 @@ static const char* __Pyx_BufFmt_DescribeTypeChar(char ch, int is_complex) { switch (ch) { + case '?': return "'bool'"; case 'c': return "'char'"; case 'b': return "'signed char'"; case 'B': return "'unsigned char'"; @@ -276,7 +348,7 @@ static size_t __Pyx_BufFmt_TypeCharToNativeSize(char ch, int is_complex) { switch (ch) { - case 'c': case 'b': case 'B': case 's': case 'p': return 1; + case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(short); case 'i': case 'I': return sizeof(int); case 'l': case 'L': return sizeof(long); @@ -305,7 +377,8 @@ typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); @@ -339,7 +412,8 @@ typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); @@ -365,7 +439,7 @@ case 'b': case 'h': case 'i': case 'l': case 'q': case 's': case 'p': return 'I'; - case 'B': case 'H': case 'I': case 'L': case 'Q': + case '?': case 'B': case 'H': case 'I': case 'L': case 'Q': return 'U'; case 'f': case 'd': case 'g': return (is_complex ? 'C' : 'R'); @@ -531,13 +605,12 @@ } /* Parse an array in the format string (e.g. (1,2,3)) */ -static CYTHON_INLINE PyObject * +static PyObject * __pyx_buffmt_parse_array(__Pyx_BufFmt_Context* ctx, const char** tsp) { const char *ts = *tsp; - int i = 0, number; - int ndim = ctx->head->field->type->ndim; -; + int i = 0, number, ndim; + ++ts; if (ctx->new_count != 1) { PyErr_SetString(PyExc_ValueError, @@ -548,6 +621,9 @@ /* Process the previous element */ if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; + // store ndim now, as field advanced by __Pyx_BufFmt_ProcessTypeChunk call + ndim = ctx->head->field->type->ndim; + /* Parse all numbers in the format string */ while (*ts && *ts != ')') { // ignore space characters (not using isspace() due to C/C++ problem on MacOS-X) @@ -611,7 +687,7 @@ ++ts; break; case '<': - if (!__Pyx_IsLittleEndian()) { + if (!__Pyx_Is_Little_Endian()) { PyErr_SetString(PyExc_ValueError, "Little-endian buffer not supported on big-endian compiler"); return NULL; } @@ -620,7 +696,7 @@ break; case '>': case '!': - if (__Pyx_IsLittleEndian()) { + if (__Pyx_Is_Little_Endian()) { PyErr_SetString(PyExc_ValueError, "Big-endian buffer not supported on little-endian compiler"); return NULL; } @@ -685,13 +761,13 @@ __Pyx_BufFmt_RaiseUnexpectedChar('Z'); return NULL; } - /* fall through */ - case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': + CYTHON_FALLTHROUGH; + case '?': case 'c': case 'b': case 'B': case 'h': case 'H': case 'i': case 'I': case 'l': case 'L': case 'q': case 'Q': case 'f': case 'd': case 'g': case 'O': case 'p': - if (ctx->enc_type == *ts && got_Z == ctx->is_complex && - ctx->enc_packmode == ctx->new_packmode) { + if ((ctx->enc_type == *ts) && (got_Z == ctx->is_complex) && + (ctx->enc_packmode == ctx->new_packmode) && (!ctx->is_valid_array)) { /* Continue pooling same type */ ctx->enc_count += ctx->new_count; ctx->new_count = 1; @@ -699,7 +775,7 @@ ++ts; break; } - /* fall through */ + CYTHON_FALLTHROUGH; case 's': /* 's' or new type (cannot be added to current pool) */ if (__Pyx_BufFmt_ProcessTypeChunk(ctx) == -1) return NULL; @@ -729,60 +805,13 @@ } } -static CYTHON_INLINE void __Pyx_ZeroBuffer(Py_buffer* buf) { - buf->buf = NULL; - buf->obj = NULL; - buf->strides = __Pyx_zeros; - buf->shape = __Pyx_zeros; - buf->suboffsets = __Pyx_minusones; -} - -static CYTHON_INLINE int __Pyx_GetBufferAndValidate( - Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, - int nd, int cast, __Pyx_BufFmt_StackElem* stack) -{ - if (obj == Py_None || obj == NULL) { - __Pyx_ZeroBuffer(buf); - return 0; - } - buf->buf = NULL; - if (__Pyx_GetBuffer(obj, buf, flags) == -1) goto fail; - if (buf->ndim != nd) { - PyErr_Format(PyExc_ValueError, - "Buffer has wrong number of dimensions (expected %d, got %d)", - nd, buf->ndim); - goto fail; - } - if (!cast) { - __Pyx_BufFmt_Context ctx; - __Pyx_BufFmt_Init(&ctx, stack, dtype); - if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail; - } - if ((unsigned)buf->itemsize != dtype->size) { - PyErr_Format(PyExc_ValueError, - "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "d byte%s) does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "d byte%s)", - buf->itemsize, (buf->itemsize > 1) ? "s" : "", - dtype->name, (Py_ssize_t)dtype->size, (dtype->size > 1) ? "s" : ""); - goto fail; - } - if (buf->suboffsets == NULL) buf->suboffsets = __Pyx_minusones; - return 0; -fail:; - __Pyx_ZeroBuffer(buf); - return -1; -} - -static CYTHON_INLINE void __Pyx_SafeReleaseBuffer(Py_buffer* info) { - if (info->buf == NULL) return; - if (info->suboffsets == __Pyx_minusones) info->suboffsets = NULL; - __Pyx_ReleaseBuffer(info); -} - /////////////// TypeInfoCompare.proto /////////////// static int __pyx_typeinfo_cmp(__Pyx_TypeInfo *a, __Pyx_TypeInfo *b); /////////////// TypeInfoCompare /////////////// -/* See if two dtypes are equal */ +//@requires: BufferFormatStructs + +// See if two dtypes are equal static int __pyx_typeinfo_cmp(__Pyx_TypeInfo *a, __Pyx_TypeInfo *b) { @@ -841,7 +870,6 @@ } - /////////////// TypeInfoToFormat.proto /////////////// struct __pyx_typeinfo_string { char string[3]; @@ -849,7 +877,9 @@ static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type); /////////////// TypeInfoToFormat /////////////// -{{# See also MemoryView.pyx:BufferFormatFromTypeInfo }} +//@requires: BufferFormatStructs + +// See also MemoryView.pyx:BufferFormatFromTypeInfo static struct __pyx_typeinfo_string __Pyx_TypeInfoToFormat(__Pyx_TypeInfo *type) { struct __pyx_typeinfo_string result = { {0} }; diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Builtins.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Builtins.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Builtins.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Builtins.c 2022-03-24 10:16:46.000000000 +0000 @@ -20,47 +20,7 @@ // access requires a rewrite as a dedicated class. static PyObject* __Pyx_Globals(void) { - Py_ssize_t i; - PyObject *names; - PyObject *globals = $moddict_cname; - Py_INCREF(globals); - names = PyObject_Dir($module_cname); - if (!names) - goto bad; - for (i = PyList_GET_SIZE(names)-1; i >= 0; i--) { -#if CYTHON_COMPILING_IN_PYPY - PyObject* name = PySequence_ITEM(names, i); - if (!name) - goto bad; -#else - PyObject* name = PyList_GET_ITEM(names, i); -#endif - if (!PyDict_Contains(globals, name)) { - PyObject* value = __Pyx_GetAttr($module_cname, name); - if (!value) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - goto bad; - } - if (PyDict_SetItem(globals, name, value) < 0) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - Py_DECREF(value); - goto bad; - } - } -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - } - Py_DECREF(names); - return globals; -bad: - Py_XDECREF(names); - Py_XDECREF(globals); - return NULL; + return __Pyx_NewRef($moddict_cname); } //////////////////// PyExecGlobals.proto //////////////////// @@ -68,17 +28,11 @@ static PyObject* __Pyx_PyExecGlobals(PyObject*); //////////////////// PyExecGlobals //////////////////// -//@requires: Globals +//@substitute: naming //@requires: PyExec static PyObject* __Pyx_PyExecGlobals(PyObject* code) { - PyObject* result; - PyObject* globals = __Pyx_Globals(); - if (unlikely(!globals)) - return NULL; - result = __Pyx_PyExec2(code, globals); - Py_DECREF(globals); - return result; + return __Pyx_PyExec2(code, $moddict_cname); } //////////////////// PyExec.proto //////////////////// @@ -100,27 +54,31 @@ if (!globals || globals == Py_None) { globals = $moddict_cname; - } else if (!PyDict_Check(globals)) { - PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s", - Py_TYPE(globals)->tp_name); + } else if (unlikely(!PyDict_Check(globals))) { + __Pyx_TypeName globals_type_name = + __Pyx_PyType_GetName(Py_TYPE(globals)); + PyErr_Format(PyExc_TypeError, + "exec() arg 2 must be a dict, not " __Pyx_FMT_TYPENAME, + globals_type_name); + __Pyx_DECREF_TypeName(globals_type_name); goto bad; } if (!locals || locals == Py_None) { locals = globals; } - if (PyDict_GetItem(globals, PYIDENT("__builtins__")) == NULL) { - if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0) + if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) { + if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)) goto bad; } if (PyCode_Check(o)) { - if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) { + if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) { PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not contain free variables"); goto bad; } - #if CYTHON_COMPILING_IN_PYPY || PY_VERSION_HEX < 0x030200B1 + #if PY_VERSION_HEX < 0x030200B1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) result = PyEval_EvalCode((PyCodeObject *)o, globals, locals); #else result = PyEval_EvalCode(o, globals, locals); @@ -128,19 +86,24 @@ } else { PyCompilerFlags cf; cf.cf_flags = 0; +#if PY_VERSION_HEX >= 0x030800A3 + cf.cf_feature_version = PY_MINOR_VERSION; +#endif if (PyUnicode_Check(o)) { cf.cf_flags = PyCF_SOURCE_IS_UTF8; s = PyUnicode_AsUTF8String(o); - if (!s) goto bad; + if (unlikely(!s)) goto bad; o = s; #if PY_MAJOR_VERSION >= 3 - } else if (!PyBytes_Check(o)) { + } else if (unlikely(!PyBytes_Check(o))) { #else - } else if (!PyString_Check(o)) { + } else if (unlikely(!PyString_Check(o))) { #endif + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); PyErr_Format(PyExc_TypeError, - "exec: arg 1 must be string, bytes or code object, got %.200s", - Py_TYPE(o)->tp_name); + "exec: arg 1 must be string, bytes or code object, got " __Pyx_FMT_TYPENAME, + o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } #if PY_MAJOR_VERSION >= 3 @@ -167,20 +130,58 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/ //////////////////// GetAttr3 //////////////////// -//@requires: ObjectHandling.c::GetAttr +//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: Exceptions.c::PyThreadStateGet +//@requires: Exceptions.c::PyErrFetchRestore +//@requires: Exceptions.c::PyErrExceptionMatches + +static PyObject *__Pyx_GetAttr3Default(PyObject *d) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(d); + return d; +} static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { - PyObject *r = __Pyx_GetAttr(o, n); - if (unlikely(!r)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) - goto bad; + PyObject *r; +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyString_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } +#endif + r = PyObject_GetAttr(o, n); + return (likely(r)) ? r : __Pyx_GetAttr3Default(d); +} + +//////////////////// HasAttr.proto //////////////////// + +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *, PyObject *); /*proto*/ + +//////////////////// HasAttr //////////////////// +//@requires: ObjectHandling.c::GetAttr + +static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { + PyObject *r; + if (unlikely(!__Pyx_PyBaseString_Check(n))) { + PyErr_SetString(PyExc_TypeError, + "hasattr(): attribute name must be string"); + return -1; + } + r = __Pyx_GetAttr(o, n); + if (!r) { PyErr_Clear(); - r = d; - Py_INCREF(d); + return 0; + } else { + Py_DECREF(r); + return 1; } - return r; -bad: - return NULL; } //////////////////// Intern.proto //////////////////// @@ -188,11 +189,12 @@ static PyObject* __Pyx_Intern(PyObject* s); /* proto */ //////////////////// Intern //////////////////// +//@requires: ObjectHandling.c::RaiseUnexpectedTypeError static PyObject* __Pyx_Intern(PyObject* s) { - if (!(likely(PyString_CheckExact(s)))) { - PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name); - return 0; + if (unlikely(!PyString_CheckExact(s))) { + __Pyx_RaiseUnexpectedTypeError("str", s); + return NULL; } Py_INCREF(s); #if PY_MAJOR_VERSION >= 3 @@ -203,46 +205,66 @@ return s; } -//////////////////// abs_int.proto //////////////////// - -static CYTHON_INLINE unsigned int __Pyx_abs_int(int x) { - if (unlikely(x == -INT_MAX-1)) - return ((unsigned int)INT_MAX) + 1U; - return (unsigned int) abs(x); -} - -//////////////////// abs_long.proto //////////////////// - -static CYTHON_INLINE unsigned long __Pyx_abs_long(long x) { - if (unlikely(x == -LONG_MAX-1)) - return ((unsigned long)LONG_MAX) + 1U; - return (unsigned long) labs(x); -} - //////////////////// abs_longlong.proto //////////////////// -static CYTHON_INLINE unsigned PY_LONG_LONG __Pyx_abs_longlong(PY_LONG_LONG x) { - if (unlikely(x == -PY_LLONG_MAX-1)) - return ((unsigned PY_LONG_LONG)PY_LLONG_MAX) + 1U; +static CYTHON_INLINE PY_LONG_LONG __Pyx_abs_longlong(PY_LONG_LONG x) { #if defined (__cplusplus) && __cplusplus >= 201103L - return (unsigned PY_LONG_LONG) std::abs(x); + return std::abs(x); #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - return (unsigned PY_LONG_LONG) llabs(x); -#elif defined (_MSC_VER) && defined (_M_X64) + return llabs(x); +#elif defined (_MSC_VER) // abs() is defined for long, but 64-bits type on MSVC is long long. - // Use MS-specific _abs64 instead. - return (unsigned PY_LONG_LONG) _abs64(x); + // Use MS-specific _abs64() instead, which returns the original (negative) value for abs(-MAX-1) + return _abs64(x); #elif defined (__GNUC__) // gcc or clang on 64 bit windows. - return (unsigned PY_LONG_LONG) __builtin_llabs(x); + return __builtin_llabs(x); #else if (sizeof(PY_LONG_LONG) <= sizeof(Py_ssize_t)) return __Pyx_sst_abs(x); - return (x<0) ? (unsigned PY_LONG_LONG)-x : (unsigned PY_LONG_LONG)x; + return (x<0) ? -x : x; #endif } +//////////////////// py_abs.proto //////////////////// + +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num);/*proto*/ + +#define __Pyx_PyNumber_Absolute(x) \ + ((likely(PyLong_CheckExact(x))) ? \ + (likely(Py_SIZE(x) >= 0) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \ + PyNumber_Absolute(x)) + +#else +#define __Pyx_PyNumber_Absolute(x) PyNumber_Absolute(x) +#endif + +//////////////////// py_abs //////////////////// + +#if CYTHON_USE_PYLONG_INTERNALS +static PyObject *__Pyx_PyLong_AbsNeg(PyObject *n) { + if (likely(Py_SIZE(n) == -1)) { + // digits are unsigned + return PyLong_FromLong(((PyLongObject*)n)->ob_digit[0]); + } +#if CYTHON_COMPILING_IN_CPYTHON + { + PyObject *copy = _PyLong_Copy((PyLongObject*)n); + if (likely(copy)) { + // negate the size to swap the sign + __Pyx_SET_SIZE(copy, -Py_SIZE(copy)); + } + return copy; + } +#else + return PyNumber_Negative(n); +#endif +} +#endif + + //////////////////// pow2.proto //////////////////// #define __Pyx_PyNumber_Power2(a, b) PyNumber_Power(a, b, Py_None) @@ -281,8 +303,11 @@ #endif } else { // FIXME: support character buffers - but CPython doesn't support them either + __Pyx_TypeName c_type_name = __Pyx_PyType_GetName(Py_TYPE(c)); PyErr_Format(PyExc_TypeError, - "ord() expected string of length 1, but %.200s found", c->ob_type->tp_name); + "ord() expected string of length 1, but " __Pyx_FMT_TYPENAME " found", + c_type_name); + __Pyx_DECREF_TypeName(c_type_name); return (long)(Py_UCS4)-1; } PyErr_Format(PyExc_TypeError, @@ -371,9 +396,6 @@ //////////////////// py_dict_viewkeys.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d); /*proto*/ //////////////////// py_dict_viewkeys //////////////////// @@ -387,9 +409,6 @@ //////////////////// py_dict_viewvalues.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d); /*proto*/ //////////////////// py_dict_viewvalues //////////////////// @@ -403,9 +422,6 @@ //////////////////// py_dict_viewitems.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/ //////////////////// py_dict_viewitems //////////////////// @@ -417,7 +433,12 @@ return CALL_UNBOUND_METHOD(PyDict_Type, "viewitems", d); } + //////////////////// pyfrozenset_new.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it); + +//////////////////// pyfrozenset_new //////////////////// //@substitute: naming static CYTHON_INLINE PyObject* __Pyx_PyFrozenSet_New(PyObject* it) { @@ -440,9 +461,9 @@ result = PyFrozenSet_New(it); if (unlikely(!result)) return NULL; - if (likely(PySet_GET_SIZE(result))) + if ((PY_VERSION_HEX >= 0x031000A1) || likely(PySet_GET_SIZE(result))) return result; - // empty frozenset is a singleton + // empty frozenset is a singleton (on Python <3.10) // seems wasteful, but CPython does the same Py_DECREF(result); #endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Capsule.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Capsule.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Capsule.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Capsule.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ -//////////////// Capsule.proto //////////////// - -/* Todo: wrap the rest of the functionality in similar functions */ -static CYTHON_INLINE PyObject *__pyx_capsule_create(void *p, const char *sig); - -//////////////// Capsule //////////////// - -static CYTHON_INLINE PyObject * -__pyx_capsule_create(void *p, CYTHON_UNUSED const char *sig) -{ - PyObject *cobj; - -#if PY_VERSION_HEX >= 0x02070000 - cobj = PyCapsule_New(p, sig, NULL); -#else - cobj = PyCObject_FromVoidPtr(p, NULL); -#endif - - return cobj; -} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CConvert.pyx cython-0.20.1+1~202203241016-9537/Cython/Utility/CConvert.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CConvert.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CConvert.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -6,19 +6,20 @@ PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") -cdef {{struct_name}} {{funcname}}(obj) except *: - cdef {{struct_name}} result +cdef {{struct_type}} {{funcname}}(obj) except *: + cdef {{struct_type}} result if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) {{for member in var_entries:}} try: value = obj['{{member.name}}'] except KeyError: raise ValueError("No value specified for struct attribute '{{member.name}}'") - result.{{member.cname}} = value + result.{{member.name}} = value {{endfor}} return result @@ -31,13 +32,14 @@ PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") -cdef {{struct_name}} {{funcname}}(obj) except *: - cdef {{struct_name}} result +cdef {{struct_type}} {{funcname}}(obj) except *: + cdef {{struct_type}} result cdef Py_ssize_t length if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) last_found = None length = len(obj) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CMath.c cython-0.20.1+1~202203241016-9537/Cython/Utility/CMath.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CMath.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CMath.c 2022-03-24 10:16:46.000000000 +0000 @@ -73,8 +73,10 @@ switch (e) { case 3: t *= b; + CYTHON_FALLTHROUGH; case 2: t *= b; + CYTHON_FALLTHROUGH; case 1: return t; case 0: diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CommonStructures.c cython-0.20.1+1~202203241016-9537/Cython/Utility/CommonStructures.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CommonStructures.c 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CommonStructures.c 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,142 @@ +/////////////// FetchSharedCythonModule.proto /////// + +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/////////////// FetchSharedCythonModule //////////// + +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (unlikely(!abi_module)) return NULL; + Py_INCREF(abi_module); + return abi_module; +} + +/////////////// FetchCommonType.proto /////////////// + +#if !CYTHON_USE_TYPE_SPECS +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); +#else +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases); +#endif + +/////////////// FetchCommonType /////////////// +//@requires:ExtensionTypes.c::FixUpExtensionType +//@requires: FetchSharedCythonModule +//@requires:StringTools.c::IncludeStringH + +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t basicsize, + Py_ssize_t expected_basicsize) { + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} + +#if !CYTHON_USE_TYPE_SPECS +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* abi_module; + const char* object_name; + PyTypeObject *cached_type = NULL; + + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + // get the final part of the object name (after the last dot) + object_name = strrchr(type->tp_name, '.'); + object_name = object_name ? object_name+1 : type->tp_name; + cached_type = (PyTypeObject*) PyObject_GetAttrString(abi_module, object_name); + if (cached_type) { + if (__Pyx_VerifyCachedType( + (PyObject *)cached_type, + object_name, + cached_type->tp_basicsize, + type->tp_basicsize) < 0) { + goto bad; + } + goto done; + } + + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(abi_module, object_name, (PyObject *)type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + +done: + Py_DECREF(abi_module); + // NOTE: always returns owned reference, or NULL on error + return cached_type; + +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} +#else + +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module, *cached_type = NULL; + // get the final part of the object name (after the last dot) + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + + cached_type = PyObject_GetAttrString(abi_module, object_name); + if (cached_type) { + Py_ssize_t basicsize; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; +#else + basicsize = likely(PyType_Check(cached_type)) ? ((PyTypeObject*) cached_type)->tp_basicsize : -1; +#endif + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + basicsize, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } + + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + // We pass the ABI module reference to avoid keeping the user module alive by foreign type usages. + (void) module; + cached_type = __Pyx_PyType_FromModuleAndSpec(abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + if (PyObject_SetAttrString(abi_module, object_name, cached_type) < 0) goto bad; + +done: + Py_DECREF(abi_module); + // NOTE: always returns owned reference, or NULL on error + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; + +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} +#endif + diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CommonTypes.c cython-0.20.1+1~202203241016-9537/Cython/Utility/CommonTypes.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CommonTypes.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CommonTypes.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,48 +0,0 @@ -/////////////// FetchCommonType.proto /////////////// - -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); - -/////////////// FetchCommonType /////////////// - -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } - -done: - Py_DECREF(fake_module); - // NOTE: always returns owned reference, or NULL on error - return cached_type; - -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; -} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Complex.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Complex.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Complex.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Complex.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ -/////////////// Header.proto.h_code /////////////// +/////////////// Header.proto /////////////// +//@proto_block: h_code #if !defined(CYTHON_CCOMPLEX) #if defined(__cplusplus) @@ -49,7 +50,8 @@ #endif -/////////////// Declarations.proto.complex_type_declarations /////////////// +/////////////// Declarations.proto /////////////// +//@proto_block: complex_type_declarations #if CYTHON_CCOMPLEX #ifdef __cplusplus @@ -186,13 +188,13 @@ return {{type_name}}_from_parts(a.real / b.real, a.imag / b.imag); } else { {{real_type}} r = b.imag / b.real; - {{real_type}} s = 1.0 / (b.real + b.imag * r); + {{real_type}} s = ({{real_type}})(1.0) / (b.real + b.imag * r); return {{type_name}}_from_parts( (a.real + a.imag * r) * s, (a.imag - a.real * r) * s); } } else { {{real_type}} r = b.real / b.imag; - {{real_type}} s = 1.0 / (b.imag + b.real * r); + {{real_type}} s = ({{real_type}})(1.0) / (b.imag + b.real * r); return {{type_name}}_from_parts( (a.real * r + a.imag) * s, (a.imag * r - a.real) * s); } @@ -251,7 +253,6 @@ case 1: return a; case 2: - z = __Pyx_c_prod{{func_suffix}}(a, a); return __Pyx_c_prod{{func_suffix}}(a, a); case 3: z = __Pyx_c_prod{{func_suffix}}(a, a); @@ -264,9 +265,17 @@ if (a.imag == 0) { if (a.real == 0) { return a; + } else if (b.imag == 0) { + z.real = pow{{m}}(a.real, b.real); + z.imag = 0; + return z; + } else if (a.real > 0) { + r = a.real; + theta = 0; + } else { + r = -a.real; + theta = atan2{{m}}(0.0, -1.0); } - r = a.real; - theta = 0; } else { r = __Pyx_c_abs{{func_suffix}}(a); theta = atan2{{m}}(a.imag, a.real); diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Coroutine.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Coroutine.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Coroutine.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Coroutine.c 2022-03-24 10:16:46.000000000 +0000 @@ -5,10 +5,20 @@ //////////////////// GeneratorYieldFrom //////////////////// //@requires: Generator +#if CYTHON_USE_TYPE_SLOTS +static void __Pyx_PyIter_CheckErrorAndDecref(PyObject *source) { + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + PyErr_Format(PyExc_TypeError, + "iter() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", source_type_name); + __Pyx_DECREF_TypeName(source_type_name); + Py_DECREF(source); +} +#endif + static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { PyObject *source_gen, *retval; #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(source)) { + if (__Pyx_Coroutine_Check(source)) { // TODO: this should only happen for types.coroutine()ed generators, but we can't determine that here Py_INCREF(source); source_gen = source; @@ -22,17 +32,19 @@ if (unlikely(!source_gen)) return NULL; if (unlikely(!PyIter_Check(source_gen))) { - PyErr_Format(PyExc_TypeError, - "iter() returned non-iterator of type '%.100s'", - Py_TYPE(source_gen)->tp_name); - Py_DECREF(source_gen); + __Pyx_PyIter_CheckErrorAndDecref(source_gen); return NULL; } } else + // CPython also allows non-iterable sequences to be iterated over #endif + { source_gen = PyObject_GetIter(source); + if (unlikely(!source_gen)) + return NULL; + } // source_gen is now the iterator, make the first next() call - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (likely(retval)) { gen->yieldfrom = source_gen; @@ -45,115 +57,55 @@ //////////////////// CoroutineYieldFrom.proto //////////////////// -#define __Pyx_Coroutine_Yield_From(gen, source) __Pyx__Coroutine_Yield_From(gen, source, 0) -static CYTHON_INLINE PyObject* __Pyx__Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source, int warn); +static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source); //////////////////// CoroutineYieldFrom //////////////////// //@requires: Coroutine //@requires: GetAwaitIter -static int __Pyx_WarnAIterDeprecation(PyObject *aiter) { - int result; -#if PY_MAJOR_VERSION >= 3 - result = PyErr_WarnFormat( - PyExc_PendingDeprecationWarning, 1, - "'%.100s' implements legacy __aiter__ protocol; " - "__aiter__ should return an asynchronous " - "iterator, not awaitable", - Py_TYPE(aiter)->tp_name); -#else - result = PyErr_WarnEx( - PyExc_PendingDeprecationWarning, - "object implements legacy __aiter__ protocol; " - "__aiter__ should return an asynchronous " - "iterator, not awaitable", - 1); -#endif - return result != 0; +static PyObject* __Pyx__Coroutine_Yield_From_Generic(__pyx_CoroutineObject *gen, PyObject *source) { + PyObject *retval; + PyObject *source_gen = __Pyx__Coroutine_GetAwaitableIter(source); + if (unlikely(!source_gen)) { + return NULL; + } + // source_gen is now the iterator, make the first next() call + if (__Pyx_Coroutine_Check(source_gen)) { + retval = __Pyx_Generator_Next(source_gen); + } else { + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); + } + if (retval) { + gen->yieldfrom = source_gen; + return retval; + } + Py_DECREF(source_gen); + return NULL; } -static CYTHON_INLINE PyObject* __Pyx__Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source, int warn) { +static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { PyObject *retval; - if (__Pyx_Coroutine_CheckExact(source)) { - if (warn && unlikely(__Pyx_WarnAIterDeprecation(source))) { - /* Warning was converted to an error. */ + if (__Pyx_Coroutine_Check(source)) { + if (unlikely(((__pyx_CoroutineObject*)source)->yieldfrom)) { + PyErr_SetString( + PyExc_RuntimeError, + "coroutine is being awaited already"); return NULL; } retval = __Pyx_Generator_Next(source); - if (retval) { - Py_INCREF(source); - gen->yieldfrom = source; - return retval; - } +#ifdef __Pyx_AsyncGen_USED + // inlined "__pyx_PyAsyncGenASend" handling to avoid the series of generic calls + } else if (__pyx_PyAsyncGenASend_CheckExact(source)) { + retval = __Pyx_async_gen_asend_iternext(source); +#endif } else { - PyObject *source_gen = __Pyx__Coroutine_GetAwaitableIter(source); - if (unlikely(!source_gen)) - return NULL; - if (warn && unlikely(__Pyx_WarnAIterDeprecation(source))) { - /* Warning was converted to an error. */ - Py_DECREF(source_gen); - return NULL; - } - // source_gen is now the iterator, make the first next() call - if (__Pyx_Coroutine_CheckExact(source_gen)) { - retval = __Pyx_Generator_Next(source_gen); - } else { - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); - } - if (retval) { - gen->yieldfrom = source_gen; - return retval; - } - Py_DECREF(source_gen); - } - return NULL; -} - - -//////////////////// CoroutineAIterYieldFrom.proto //////////////////// - -static CYTHON_INLINE PyObject* __Pyx_Coroutine_AIter_Yield_From(__pyx_CoroutineObject *gen, PyObject *source); - -//////////////////// CoroutineAIterYieldFrom //////////////////// -//@requires: CoroutineYieldFrom - -static CYTHON_INLINE PyObject* __Pyx_Coroutine_AIter_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { -#if CYTHON_USE_ASYNC_SLOTS - __Pyx_PyAsyncMethodsStruct* am = __Pyx_PyType_AsAsync(source); - if (likely(am && am->am_anext)) { - // Starting with CPython 3.5.2, __aiter__ should return - // asynchronous iterators directly (not awaitables that - // resolve to asynchronous iterators.) - // - // Therefore, we check if the object that was returned - // from __aiter__ has an __anext__ method. If it does, - // we return it directly as StopIteration result, - // which avoids yielding. - // - // See http://bugs.python.org/issue27243 for more - // details. - PyErr_SetObject(PyExc_StopIteration, source); - return NULL; + return __Pyx__Coroutine_Yield_From_Generic(gen, source); } -#endif -#if PY_VERSION_HEX < 0x030500B2 - if (!__Pyx_PyType_AsAsync(source)) { - #ifdef __Pyx_Coroutine_USED - if (!__Pyx_Coroutine_CheckExact(source)) // quickly rule out a likely case - #endif - { - // same as above in slow - PyObject *method = __Pyx_PyObject_GetAttrStr(source, PYIDENT("__anext__")); - if (method) { - Py_DECREF(method); - PyErr_SetObject(PyExc_StopIteration, source); - return NULL; - } - PyErr_Clear(); - } + if (retval) { + Py_INCREF(source); + gen->yieldfrom = source; } -#endif - return __Pyx__Coroutine_Yield_From(gen, source, 1); + return retval; } @@ -163,20 +115,54 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *o); /*proto*/ //////////////////// GetAwaitIter //////////////////// -//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetMethod //@requires: ObjectHandling.c::PyObjectCallNoArg //@requires: ObjectHandling.c::PyObjectCallOneArg static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) { #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(o)) { - Py_INCREF(o); - return o; + if (__Pyx_Coroutine_Check(o)) { + return __Pyx_NewRef(o); } #endif return __Pyx__Coroutine_GetAwaitableIter(o); } + +static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { +#if PY_VERSION_HEX >= 0x030600B3 || defined(_PyErr_FormatFromCause) + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + _PyErr_FormatFromCause(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); +#elif PY_MAJOR_VERSION >= 3 + PyObject *exc, *val, *val2, *tb; + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + assert(PyErr_Occurred()); + PyErr_Fetch(&exc, &val, &tb); + PyErr_NormalizeException(&exc, &val, &tb); + if (tb != NULL) { + PyException_SetTraceback(val, tb); + Py_DECREF(tb); + } + Py_DECREF(exc); + assert(!PyErr_Occurred()); + PyErr_Format(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); + + PyErr_Fetch(&exc, &val2, &tb); + PyErr_NormalizeException(&exc, &val2, &tb); + Py_INCREF(val); + PyException_SetCause(val2, val); + PyException_SetContext(val2, val); + PyErr_Restore(exc, val2, tb); +#else + // since Py2 does not have exception chaining, it's better to avoid shadowing exceptions there + source++; +#endif +} + // adapted from genobject.c in Py3.5 static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { PyObject *res; @@ -188,43 +174,41 @@ #endif #if PY_VERSION_HEX >= 0x030500B2 || defined(PyCoro_CheckExact) if (PyCoro_CheckExact(obj)) { - Py_INCREF(obj); - return obj; + return __Pyx_NewRef(obj); } else #endif #if CYTHON_COMPILING_IN_CPYTHON && defined(CO_ITERABLE_COROUTINE) if (PyGen_CheckExact(obj) && ((PyGenObject*)obj)->gi_code && ((PyCodeObject *)((PyGenObject*)obj)->gi_code)->co_flags & CO_ITERABLE_COROUTINE) { // Python generator marked with "@types.coroutine" decorator - Py_INCREF(obj); - return obj; + return __Pyx_NewRef(obj); } else #endif { - PyObject *method = __Pyx_PyObject_GetAttrStr(obj, PYIDENT("__await__")); - if (unlikely(!method)) goto slot_error; - #if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *function = PyMethod_GET_FUNCTION(method); - res = __Pyx_PyObject_CallOneArg(function, self); - } else - res = __Pyx_PyObject_CallNoArg(method); - } else - #endif + PyObject *method = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, PYIDENT("__await__"), &method); + if (likely(is_method)) { + res = __Pyx_PyObject_CallOneArg(method, obj); + } else if (likely(method)) { res = __Pyx_PyObject_CallNoArg(method); + } else + goto slot_error; Py_DECREF(method); } - if (unlikely(!res)) goto bad; - if (!PyIter_Check(res)) { + if (unlikely(!res)) { + // surprisingly, CPython replaces the exception here... + __Pyx_Coroutine_AwaitableIterError(obj); + goto bad; + } + if (unlikely(!PyIter_Check(res))) { + __Pyx_TypeName res_type_name = __Pyx_PyType_GetName(Py_TYPE(res)); PyErr_Format(PyExc_TypeError, - "__await__() returned non-iterator of type '%.100s'", - Py_TYPE(res)->tp_name); + "__await__() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", res_type_name); + __Pyx_DECREF_TypeName(res_type_name); Py_CLEAR(res); } else { int is_coroutine = 0; #ifdef __Pyx_Coroutine_USED - is_coroutine |= __Pyx_Coroutine_CheckExact(res); + is_coroutine |= __Pyx_Coroutine_Check(res); #endif #if PY_VERSION_HEX >= 0x030500B2 || defined(PyCoro_CheckExact) is_coroutine |= PyCoro_CheckExact(res); @@ -239,9 +223,12 @@ } return res; slot_error: - PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "object " __Pyx_FMT_TYPENAME " can't be used in 'await' expression", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } bad: return NULL; } @@ -256,13 +243,8 @@ //@requires: GetAwaitIter //@requires: ObjectHandling.c::PyObjectCallMethod0 -static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAsyncIter(PyObject *obj) { -#if CYTHON_USE_ASYNC_SLOTS - __Pyx_PyAsyncMethodsStruct* am = __Pyx_PyType_AsAsync(obj); - if (likely(am && am->am_aiter)) { - return (*am->am_aiter)(obj); - } -#endif +static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { + __Pyx_TypeName obj_type_name; #if PY_VERSION_HEX < 0x030500B1 { PyObject *iter = __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); @@ -274,21 +256,36 @@ } #else // avoid C warning about 'unused function' - if (0) (void) __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); + if ((0)) (void) __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __aiter__ method, got %.100s", - Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __aiter__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } -static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *obj) { + +static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAsyncIter(PyObject *obj) { +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(obj)) { + return __Pyx_NewRef(obj); + } +#endif #if CYTHON_USE_ASYNC_SLOTS - __Pyx_PyAsyncMethodsStruct* am = __Pyx_PyType_AsAsync(obj); - if (likely(am && am->am_anext)) { - return (*am->am_anext)(obj); + { + __Pyx_PyAsyncMethodsStruct* am = __Pyx_PyType_AsAsync(obj); + if (likely(am && am->am_aiter)) { + return (*am->am_aiter)(obj); + } } #endif + return __Pyx_Coroutine_GetAsyncIter_Generic(obj); +} + + +static PyObject *__Pyx__Coroutine_AsyncIterNext(PyObject *obj) { #if PY_VERSION_HEX < 0x030500B1 { PyObject *value = __Pyx_PyObject_CallMethod0(obj, PYIDENT("__anext__")); @@ -298,92 +295,192 @@ // FIXME: for the sake of a nicely conforming exception message, assume any AttributeError meant '__anext__' if (PyErr_ExceptionMatches(PyExc_AttributeError)) #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __anext__ method, got %.100s", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __anext__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } return NULL; } +static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *obj) { +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(obj)) { + return __Pyx_async_gen_anext(obj); + } +#endif +#if CYTHON_USE_ASYNC_SLOTS + { + __Pyx_PyAsyncMethodsStruct* am = __Pyx_PyType_AsAsync(obj); + if (likely(am && am->am_anext)) { + return (*am->am_anext)(obj); + } + } +#endif + return __Pyx__Coroutine_AsyncIterNext(obj); +} + + //////////////////// pep479.proto //////////////////// -static void __Pyx_Generator_Replace_StopIteration(void); /*proto*/ +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen); /*proto*/ //////////////////// pep479 //////////////////// //@requires: Exceptions.c::GetException -static void __Pyx_Generator_Replace_StopIteration(void) { - PyObject *exc, *val, *tb; - // Chain exceptions by moving StopIteration to exc_info before creating the RuntimeError. - // In Py2.x, no chaining happens, but the exception still stays visible in exc_info. +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen) { + PyObject *exc, *val, *tb, *cur_exc; __Pyx_PyThreadState_declare + #ifdef __Pyx_StopAsyncIteration_USED + int is_async_stopiteration = 0; + #endif + CYTHON_MAYBE_UNUSED_VAR(in_async_gen); + + cur_exc = PyErr_Occurred(); + if (likely(!__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) { + #ifdef __Pyx_StopAsyncIteration_USED + if (in_async_gen && unlikely(__Pyx_PyErr_GivenExceptionMatches(cur_exc, __Pyx_PyExc_StopAsyncIteration))) { + is_async_stopiteration = 1; + } else + #endif + return; + } + __Pyx_PyThreadState_assign + // Chain exceptions by moving Stop(Async)Iteration to exc_info before creating the RuntimeError. + // In Py2.x, no chaining happens, but the exception still stays visible in exc_info. __Pyx_GetException(&exc, &val, &tb); Py_XDECREF(exc); Py_XDECREF(val); Py_XDECREF(tb); - PyErr_SetString(PyExc_RuntimeError, "generator raised StopIteration"); + PyErr_SetString(PyExc_RuntimeError, + #ifdef __Pyx_StopAsyncIteration_USED + is_async_stopiteration ? "async generator raised StopAsyncIteration" : + in_async_gen ? "async generator raised StopIteration" : + #endif + "generator raised StopIteration"); } //////////////////// CoroutineBase.proto //////////////////// +//@substitute: naming -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyObject *); +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); +#if CYTHON_USE_EXC_INFO_STACK +// See https://bugs.python.org/issue25612 +#define __Pyx_ExcInfoStruct _PyErr_StackItem +#else +// Minimal replacement struct for Py<3.7, without the Py3.7 exception state stack. typedef struct { - PyObject_HEAD - __pyx_coroutine_body_t body; - PyObject *closure; PyObject *exc_type; PyObject *exc_value; PyObject *exc_traceback; +} __Pyx_ExcInfoStruct; +#endif + +typedef struct __pyx_CoroutineObject { + PyObject_HEAD + __pyx_coroutine_body_t body; + PyObject *closure; + __Pyx_ExcInfoStruct gi_exc_state; PyObject *gi_weakreflist; PyObject *classobj; PyObject *yieldfrom; PyObject *gi_name; PyObject *gi_qualname; PyObject *gi_modulename; + PyObject *gi_code; + PyObject *gi_frame; int resume_label; // using T_BOOL for property below requires char value char is_running; } __pyx_CoroutineObject; static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *closure, + PyTypeObject *type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, PyObject *name, PyObject *qualname, PyObject *module_name); /*proto*/ + +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name); /*proto*/ + +static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *self); static int __Pyx_Coroutine_clear(PyObject *self); /*proto*/ +static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); /*proto*/ +static PyObject *__Pyx_Coroutine_Close(PyObject *self); /*proto*/ +static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); /*proto*/ -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue); /*proto*/ +// macros for exception state swapping instead of inline functions to make use of the local thread state context +#if CYTHON_USE_EXC_INFO_STACK +#define __Pyx_Coroutine_SwapException(self) +#define __Pyx_Coroutine_ResetAndClearException(self) __Pyx_Coroutine_ExceptionClear(&(self)->gi_exc_state) #else -#define __Pyx_PyGen_FetchStopIterationValue(pvalue) PyGen_FetchStopIterationValue(pvalue) +#define __Pyx_Coroutine_SwapException(self) { \ + __Pyx_ExceptionSwap(&(self)->gi_exc_state.exc_type, &(self)->gi_exc_state.exc_value, &(self)->gi_exc_state.exc_traceback); \ + __Pyx_Coroutine_ResetFrameBackpointer(&(self)->gi_exc_state); \ + } +#define __Pyx_Coroutine_ResetAndClearException(self) { \ + __Pyx_ExceptionReset((self)->gi_exc_state.exc_type, (self)->gi_exc_state.exc_value, (self)->gi_exc_state.exc_traceback); \ + (self)->gi_exc_state.exc_type = (self)->gi_exc_state.exc_value = (self)->gi_exc_state.exc_traceback = NULL; \ + } #endif +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \ + __Pyx_PyGen__FetchStopIterationValue($local_tstate_cname, pvalue) +#else +#define __Pyx_PyGen_FetchStopIterationValue(pvalue) \ + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, pvalue) +#endif +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *tstate, PyObject **pvalue); /*proto*/ +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state); /*proto*/ + //////////////////// Coroutine.proto //////////////////// #define __Pyx_Coroutine_USED static PyTypeObject *__pyx_CoroutineType = 0; static PyTypeObject *__pyx_CoroutineAwaitType = 0; -#define __Pyx_Coroutine_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineType) +#define __Pyx_Coroutine_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineType) +// __Pyx_Coroutine_Check(obj): see override for IterableCoroutine below +#define __Pyx_Coroutine_Check(obj) __Pyx_Coroutine_CheckExact(obj) +#define __Pyx_CoroutineAwait_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineAwaitType) -#define __Pyx_Coroutine_New(body, closure, name, qualname, module_name) \ - __Pyx__Coroutine_New(__pyx_CoroutineType, body, closure, name, qualname, module_name) +#define __Pyx_Coroutine_New(body, code, closure, name, qualname, module_name) \ + __Pyx__Coroutine_New(__pyx_CoroutineType, body, code, closure, name, qualname, module_name) -static int __pyx_Coroutine_init(void); /*proto*/ +static int __pyx_Coroutine_init(PyObject *module); /*proto*/ static PyObject *__Pyx__Coroutine_await(PyObject *coroutine); /*proto*/ +typedef struct { + PyObject_HEAD + PyObject *coroutine; +} __pyx_CoroutineAwaitObject; + +static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, PyObject *arg); /*proto*/ +static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, PyObject *args); /*proto*/ + //////////////////// Generator.proto //////////////////// #define __Pyx_Generator_USED static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_GeneratorType) -#define __Pyx_Generator_New(body, closure, name, qualname, module_name) \ - __Pyx__Coroutine_New(__pyx_GeneratorType, body, closure, name, qualname, module_name) +#define __Pyx_Generator_New(body, code, closure, name, qualname, module_name) \ + __Pyx__Coroutine_New(__pyx_GeneratorType, body, code, closure, name, qualname, module_name) static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); /*proto*/ +static int __pyx_Generator_init(PyObject *module); /*proto*/ + + +//////////////////// AsyncGen //////////////////// +//@requires: AsyncGen.c::AsyncGenerator +// -> empty, only delegates to separate file //////////////////// CoroutineBase //////////////////// @@ -392,16 +489,22 @@ //@requires: Exceptions.c::PyThreadStateGet //@requires: Exceptions.c::SwapException //@requires: Exceptions.c::RaiseException +//@requires: Exceptions.c::SaveResetException //@requires: ObjectHandling.c::PyObjectCallMethod1 +//@requires: ObjectHandling.c::PyObjectCallNoArg +//@requires: ObjectHandling.c::PyObjectFastCall //@requires: ObjectHandling.c::PyObjectGetAttrStr -//@requires: CommonTypes.c::FetchCommonType +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError +//@requires: CommonStructures.c::FetchCommonType +//@requires: ModuleSetupCode.c::IncludeStructmemberH -#include #include - -static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); -static PyObject *__Pyx_Coroutine_Close(PyObject *self); -static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); +#if PY_VERSION_HEX >= 0x030b00a6 + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif #define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) @@ -411,12 +514,10 @@ // Returns 0 if no exception or StopIteration is set. // If any other exception is set, returns -1 and leaves // pvalue unchanged. -#if 1 || PY_VERSION_HEX < 0x030300B0 -static int __Pyx_PyGen_FetchStopIterationValue(PyObject **pvalue) { +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *$local_tstate_cname, PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign + CYTHON_UNUSED_VAR($local_tstate_cname); __Pyx_ErrFetch(&et, &ev, &tb); @@ -435,7 +536,7 @@ value = Py_None; } #if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { value = ((PyStopIterationObject *)ev)->value; Py_INCREF(value); Py_DECREF(ev); @@ -457,7 +558,7 @@ } Py_DECREF(ev); } - else if (!PyObject_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { + else if (!__Pyx_TypeCheck(ev, (PyTypeObject*)PyExc_StopIteration)) { // 'steal' reference to ev value = ev; } @@ -467,7 +568,7 @@ *pvalue = value; return 0; } - } else if (!PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { + } else if (!__Pyx_PyErr_GivenExceptionMatches(et, PyExc_StopIteration)) { __Pyx_ErrRestore(et, ev, tb); return -1; } @@ -503,119 +604,277 @@ *pvalue = value; return 0; } -#endif static CYTHON_INLINE -void __Pyx_Coroutine_ExceptionClear(__pyx_CoroutineObject *self) { - PyObject *exc_type = self->exc_type; - PyObject *exc_value = self->exc_value; - PyObject *exc_traceback = self->exc_traceback; - - self->exc_type = NULL; - self->exc_value = NULL; - self->exc_traceback = NULL; - - Py_XDECREF(exc_type); - Py_XDECREF(exc_value); - Py_XDECREF(exc_traceback); +void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_CLEAR(exc_state->exc_value); +#else + PyObject *t, *v, *tb; + t = exc_state->exc_type; + v = exc_state->exc_value; + tb = exc_state->exc_traceback; + + exc_state->exc_type = NULL; + exc_state->exc_value = NULL; + exc_state->exc_traceback = NULL; + + Py_XDECREF(t); + Py_XDECREF(v); + Py_XDECREF(tb); +#endif } -static CYTHON_INLINE -int __Pyx_Coroutine_CheckRunning(__pyx_CoroutineObject *gen) { - if (unlikely(gen->is_running)) { - PyErr_SetString(PyExc_ValueError, - "generator already executing"); - return 1; +#define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) { + const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check((PyObject*)gen)) { + msg = "coroutine already executing"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact((PyObject*)gen)) { + msg = "async generator already executing"; + #endif + } else { + msg = "generator already executing"; } - return 0; + PyErr_SetString(PyExc_ValueError, msg); } -static CYTHON_INLINE -PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value) { - PyObject *retval; +#define __Pyx_Coroutine_NotStartedError(gen) (__Pyx__Coroutine_NotStartedError(gen), (PyObject*)NULL) +static void __Pyx__Coroutine_NotStartedError(PyObject *gen) { + const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check(gen)) { + msg = "can't send non-None value to a just-started coroutine"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(gen)) { + msg = "can't send non-None value to a just-started async generator"; + #endif + } else { + msg = "can't send non-None value to a just-started generator"; + } + PyErr_SetString(PyExc_TypeError, msg); +} + +#define __Pyx_Coroutine_AlreadyTerminatedError(gen, value, closing) (__Pyx__Coroutine_AlreadyTerminatedError(gen, value, closing), (PyObject*)NULL) +static void __Pyx__Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) { + CYTHON_MAYBE_UNUSED_VAR(gen); + CYTHON_MAYBE_UNUSED_VAR(closing); + #ifdef __Pyx_Coroutine_USED + if (!closing && __Pyx_Coroutine_Check(gen)) { + // `self` is an exhausted coroutine: raise an error, + // except when called from gen_close(), which should + // always be a silent method. + PyErr_SetString(PyExc_RuntimeError, "cannot reuse already awaited coroutine"); + } else + #endif + if (value) { + // `gen` is an exhausted generator: + // only set exception if called from send(). + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); + else + #endif + PyErr_SetNone(PyExc_StopIteration); + } +} + +static +PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, int closing) { __Pyx_PyThreadState_declare + PyThreadState *tstate; + __Pyx_ExcInfoStruct *exc_state; + PyObject *retval; assert(!self->is_running); if (unlikely(self->resume_label == 0)) { if (unlikely(value && value != Py_None)) { - PyErr_SetString(PyExc_TypeError, - "can't send non-None value to a " - "just-started generator"); - return NULL; + return __Pyx_Coroutine_NotStartedError((PyObject*)self); } } if (unlikely(self->resume_label == -1)) { - PyErr_SetNone(PyExc_StopIteration); - return NULL; + return __Pyx_Coroutine_AlreadyTerminatedError((PyObject*)self, value, closing); } +#if CYTHON_FAST_THREAD_STATE __Pyx_PyThreadState_assign - if (value) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON - // FIXME: what to do in PyPy? + tstate = $local_tstate_cname; #else + tstate = __Pyx_PyThreadState_Current; +#endif + + // Traceback/Frame rules pre-Py3.7: + // - on entry, save external exception state in self->gi_exc_state, restore it on exit + // - on exit, keep internally generated exceptions in self->gi_exc_state, clear everything else + // - on entry, set "f_back" pointer of internal exception traceback to (current) outer call frame + // - on exit, clear "f_back" of internal exception traceback + // - do not touch external frames and tracebacks + + // Traceback/Frame rules for Py3.7+ (CYTHON_USE_EXC_INFO_STACK): + // - on entry, push internal exception state in self->gi_exc_state on the exception stack + // - on exit, keep internally generated exceptions in self->gi_exc_state, clear everything else + // - on entry, set "f_back" pointer of internal exception traceback to (current) outer call frame + // - on exit, clear "f_back" of internal exception traceback + // - do not touch external frames and tracebacks + + exc_state = &self->gi_exc_state; + if (exc_state->exc_value) { + #if CYTHON_COMPILING_IN_PYPY + // FIXME: what to do in PyPy? + #else // Generators always return to their most recent caller, not // necessarily their creator. - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 + // owned reference! + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + if (exc_tb) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; PyFrameObject *f = tb->tb_frame; - Py_XINCREF($local_tstate_cname->frame); assert(f->f_back == NULL); - f->f_back = $local_tstate_cname->frame; + #if PY_VERSION_HEX >= 0x030B00A1 + // PyThreadState_GetFrame returns NULL if there isn't a current frame + // which is a valid state so no need to check + f->f_back = PyThreadState_GetFrame(tstate); + #else + Py_XINCREF(tstate->frame); + f->f_back = tstate->frame; + #endif + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif } -#endif - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); + #endif + } + +#if CYTHON_USE_EXC_INFO_STACK + // See https://bugs.python.org/issue25612 + exc_state->previous_item = tstate->exc_info; + tstate->exc_info = exc_state; +#else + if (exc_state->exc_type) { + // We were in an except handler when we left, + // restore the exception state which was put aside. + __Pyx_ExceptionSwap(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); + // self->exc_* now holds the exception state of the caller } else { - __Pyx_Coroutine_ExceptionClear(self); + // save away the exception state of the caller + __Pyx_Coroutine_ExceptionClear(exc_state); + __Pyx_ExceptionSave(&exc_state->exc_type, &exc_state->exc_value, &exc_state->exc_traceback); } +#endif self->is_running = 1; - retval = self->body((PyObject *) self, value); + retval = self->body(self, tstate, value); self->is_running = 0; - if (retval) { - __Pyx_ExceptionSwap(&self->exc_type, &self->exc_value, - &self->exc_traceback); -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON - // FIXME: what to do in PyPy? -#else - // Don't keep the reference to f_back any longer than necessary. It - // may keep a chain of frames alive or it could create a reference - // cycle. - if (self->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) self->exc_traceback; - PyFrameObject *f = tb->tb_frame; - Py_CLEAR(f->f_back); - } +#if CYTHON_USE_EXC_INFO_STACK + // See https://bugs.python.org/issue25612 + exc_state = &self->gi_exc_state; + tstate->exc_info = exc_state->previous_item; + exc_state->previous_item = NULL; + // Cut off the exception frame chain so that we can reconnect it on re-entry above. + __Pyx_Coroutine_ResetFrameBackpointer(exc_state); #endif - } else { - __Pyx_Coroutine_ExceptionClear(self); - } return retval; } +static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStruct *exc_state) { + // Don't keep the reference to f_back any longer than necessary. It + // may keep a chain of frames alive or it could create a reference + // cycle. +#if CYTHON_COMPILING_IN_PYPY + // FIXME: what to do in PyPy? +#else + PyObject *exc_tb; + + #if PY_VERSION_HEX >= 0x030B00a4 + if (!exc_state->exc_value) return; + // owned reference! + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + + if (likely(exc_tb)) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; + PyFrameObject *f = tb->tb_frame; + Py_CLEAR(f->f_back); + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif + } +#endif +} + static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(PyObject *retval) { - if (unlikely(!retval && !PyErr_Occurred())) { - // method call must not terminate with NULL without setting an exception - PyErr_SetNone(PyExc_StopIteration); +PyObject *__Pyx_Coroutine_MethodReturn(PyObject* gen, PyObject *retval) { + CYTHON_MAYBE_UNUSED_VAR(gen); + if (unlikely(!retval)) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (!__Pyx_PyErr_Occurred()) { + // method call must not terminate with NULL without setting an exception + PyObject *exc = PyExc_StopIteration; + #ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(gen)) + exc = __Pyx_PyExc_StopAsyncIteration; + #endif + __Pyx_PyErr_SetNone(exc); + } } return retval; } +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03030000 && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) +static CYTHON_INLINE +PyObject *__Pyx_PyGen_Send(PyGenObject *gen, PyObject *arg) { +#if PY_VERSION_HEX <= 0x030A00A1 + return _PyGen_Send(gen, arg); +#else + PyObject *result; + // PyIter_Send() asserts non-NULL arg + if (PyIter_Send((PyObject*)gen, arg ? arg : Py_None, &result) == PYGEN_RETURN) { + if (PyAsyncGen_CheckExact(gen)) { + assert(result == Py_None); + PyErr_SetNone(PyExc_StopAsyncIteration); + } + else if (result == Py_None) { + PyErr_SetNone(PyExc_StopIteration); + } + else { + _PyGen_SetStopIterationValue(result); + } + Py_CLEAR(result); + } + return result; +#endif +} +#endif + static CYTHON_INLINE PyObject *__Pyx_Coroutine_FinishDelegation(__pyx_CoroutineObject *gen) { PyObject *ret; PyObject *val = NULL; __Pyx_Coroutine_Undelegate(gen); - __Pyx_PyGen_FetchStopIterationValue(&val); + __Pyx_PyGen__FetchStopIterationValue(__Pyx_PyThreadState_Current, &val); // val == NULL on failure => pass on exception - ret = __Pyx_Coroutine_SendEx(gen, val); + ret = __Pyx_Coroutine_SendEx(gen, val, 0); Py_XDECREF(val); return ret; } @@ -624,8 +883,8 @@ PyObject *retval; __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; // FIXME: does this really need an INCREF() ? @@ -637,13 +896,30 @@ } else #endif #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { + if (__Pyx_Coroutine_Check(yf)) { ret = __Pyx_Coroutine_Send(yf, value); } else #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + ret = __Pyx_async_gen_asend_send(yf, value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03030000 && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + // _PyGen_Send() is not exported before Py3.6 + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03050000 && defined(PyCoro_CheckExact) && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + // _PyGen_Send() is not exported before Py3.6 + if (PyCoro_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, value == Py_None ? NULL : value); + } else + #endif { if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); else ret = __Pyx_PyObject_CallMethod1(yf, PYIDENT("send"), value); } @@ -654,9 +930,9 @@ } retval = __Pyx_Coroutine_FinishDelegation(gen); } else { - retval = __Pyx_Coroutine_SendEx(gen, value); + retval = __Pyx_Coroutine_SendEx(gen, value, 0); } - return __Pyx_Coroutine_MethodReturn(retval); + return __Pyx_Coroutine_MethodReturn(self, retval); } // This helper function is used by gen_close and gen_throw to @@ -673,25 +949,39 @@ } else #endif #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { + if (__Pyx_Coroutine_Check(yf)) { retval = __Pyx_Coroutine_Close(yf); if (!retval) return -1; } else + if (__Pyx_CoroutineAwait_CheckExact(yf)) { + retval = __Pyx_CoroutineAwait_Close((__pyx_CoroutineAwaitObject*)yf, NULL); + if (!retval) + return -1; + } else + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_PyAsyncGenASend_CheckExact(yf)) { + retval = __Pyx_async_gen_asend_close(yf, NULL); + // cannot fail + } else + if (__pyx_PyAsyncGenAThrow_CheckExact(yf)) { + retval = __Pyx_async_gen_athrow_close(yf, NULL); + // cannot fail + } else #endif { PyObject *meth; gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("close")); + meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("close")); if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { PyErr_WriteUnraisable(yf); } - PyErr_Clear(); } else { - retval = PyObject_CallFunction(meth, NULL); + retval = __Pyx_PyObject_CallNoArg(meth); Py_DECREF(meth); - if (!retval) + if (unlikely(!retval)) err = -1; } gen->is_running = 0; @@ -703,8 +993,8 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self; PyObject *yf = gen->yieldfrom; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; // FIXME: does this really need an INCREF() ? @@ -716,7 +1006,18 @@ ret = __Pyx_Generator_Next(yf); } else #endif - ret = Py_TYPE(yf)->tp_iternext(yf); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03030000 && (defined(__linux__) || PY_VERSION_HEX >= 0x030600B3) + // _PyGen_Send() is not exported before Py3.6 + if (PyGen_CheckExact(yf)) { + ret = __Pyx_PyGen_Send((PyGenObject*)yf, NULL); + } else + #endif + #ifdef __Pyx_Coroutine_USED + if (__Pyx_Coroutine_Check(yf)) { + ret = __Pyx_Coroutine_Send(yf, Py_None); + } else + #endif + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); gen->is_running = 0; //Py_DECREF(yf); if (likely(ret)) { @@ -724,7 +1025,12 @@ } return __Pyx_Coroutine_FinishDelegation(gen); } - return __Pyx_Coroutine_SendEx(gen, Py_None); + return __Pyx_Coroutine_SendEx(gen, Py_None, 0); +} + +static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + return __Pyx_Coroutine_Close(self); } static PyObject *__Pyx_Coroutine_Close(PyObject *self) { @@ -733,8 +1039,8 @@ PyObject *yf = gen->yieldfrom; int err = 0; - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { Py_INCREF(yf); @@ -744,20 +1050,31 @@ } if (err == 0) PyErr_SetNone(PyExc_GeneratorExit); - retval = __Pyx_Coroutine_SendEx(gen, NULL); - if (retval) { + retval = __Pyx_Coroutine_SendEx(gen, NULL, 1); + if (unlikely(retval)) { + const char *msg; Py_DECREF(retval); - PyErr_SetString(PyExc_RuntimeError, - "generator ignored GeneratorExit"); + if ((0)) { + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_Coroutine_Check(self)) { + msg = "coroutine ignored GeneratorExit"; + #endif + #ifdef __Pyx_AsyncGen_USED + } else if (__Pyx_AsyncGen_CheckExact(self)) { +#if PY_VERSION_HEX < 0x03060000 + msg = "async generator ignored GeneratorExit - might require Python 3.6+ finalisation (PEP 525)"; +#else + msg = "async generator ignored GeneratorExit"; +#endif + #endif + } else { + msg = "generator ignored GeneratorExit"; + } + PyErr_SetString(PyExc_RuntimeError, msg); return NULL; } raised_exception = PyErr_Occurred(); - if (!raised_exception - || raised_exception == PyExc_StopIteration - || raised_exception == PyExc_GeneratorExit - || PyErr_GivenExceptionMatches(raised_exception, PyExc_GeneratorExit) - || PyErr_GivenExceptionMatches(raised_exception, PyExc_StopIteration)) - { + if (likely(!raised_exception || __Pyx_PyErr_GivenExceptionMatches2(raised_exception, PyExc_GeneratorExit, PyExc_StopIteration))) { // ignore these errors if (raised_exception) PyErr_Clear(); Py_INCREF(Py_None); @@ -766,55 +1083,61 @@ return NULL; } -static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { +static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject *val, PyObject *tb, + PyObject *args, int close_on_genexit) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - PyObject *typ; - PyObject *tb = NULL; - PyObject *val = NULL; PyObject *yf = gen->yieldfrom; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) - return NULL; - - if (unlikely(__Pyx_Coroutine_CheckRunning(gen))) - return NULL; + if (unlikely(gen->is_running)) + return __Pyx_Coroutine_AlreadyRunningError(gen); if (yf) { PyObject *ret; Py_INCREF(yf); - if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) { + if (__Pyx_PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit) && close_on_genexit) { + // Asynchronous generators *should not* be closed right away. + // We have to allow some awaits to work it through, hence the + // `close_on_genexit` parameter here. int err = __Pyx_Coroutine_CloseIter(gen, yf); Py_DECREF(yf); __Pyx_Coroutine_Undelegate(gen); if (err < 0) - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0)); goto throw_here; } gen->is_running = 1; + if (0 #ifdef __Pyx_Generator_USED - if (__Pyx_Generator_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else + || __Pyx_Generator_CheckExact(yf) #endif #ifdef __Pyx_Coroutine_USED - if (__Pyx_Coroutine_CheckExact(yf)) { - ret = __Pyx_Coroutine_Throw(yf, args); - } else + || __Pyx_Coroutine_Check(yf) #endif - { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("throw")); + ) { + ret = __Pyx__Coroutine_Throw(yf, typ, val, tb, args, close_on_genexit); + #ifdef __Pyx_Coroutine_USED + } else if (__Pyx_CoroutineAwait_CheckExact(yf)) { + ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); + #endif + } else { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("throw")); if (unlikely(!meth)) { Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { gen->is_running = 0; return NULL; } - PyErr_Clear(); __Pyx_Coroutine_Undelegate(gen); gen->is_running = 0; goto throw_here; } - ret = PyObject_CallObject(meth, args); + if (likely(args)) { + ret = __Pyx_PyObject_Call(meth, args, NULL); + } else { + // "tb" or even "val" might be NULL, but that also correctly terminates the argument list + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } Py_DECREF(meth); } gen->is_running = 0; @@ -822,23 +1145,40 @@ if (!ret) { ret = __Pyx_Coroutine_FinishDelegation(gen); } - return __Pyx_Coroutine_MethodReturn(ret); + return __Pyx_Coroutine_MethodReturn(self, ret); } throw_here: __Pyx_Raise(typ, val, tb, NULL); - return __Pyx_Coroutine_MethodReturn(__Pyx_Coroutine_SendEx(gen, NULL)); + return __Pyx_Coroutine_MethodReturn(self, __Pyx_Coroutine_SendEx(gen, NULL, 0)); } -static int __Pyx_Coroutine_traverse(PyObject *self, visitproc visit, void *arg) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; +static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { + PyObject *typ; + PyObject *val = NULL; + PyObject *tb = NULL; + + if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))) + return NULL; + return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); +} + +static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_VISIT(exc_state->exc_value); +#else + Py_VISIT(exc_state->exc_type); + Py_VISIT(exc_state->exc_value); + Py_VISIT(exc_state->exc_traceback); +#endif + return 0; +} + +static int __Pyx_Coroutine_traverse(__pyx_CoroutineObject *gen, visitproc visit, void *arg) { Py_VISIT(gen->closure); Py_VISIT(gen->classobj); Py_VISIT(gen->yieldfrom); - Py_VISIT(gen->exc_type); - Py_VISIT(gen->exc_value); - Py_VISIT(gen->exc_traceback); - return 0; + return __Pyx_Coroutine_traverse_excstate(&gen->gi_exc_state, visit, arg); } static int __Pyx_Coroutine_clear(PyObject *self) { @@ -847,11 +1187,17 @@ Py_CLEAR(gen->closure); Py_CLEAR(gen->classobj); Py_CLEAR(gen->yieldfrom); - Py_CLEAR(gen->exc_type); - Py_CLEAR(gen->exc_value); - Py_CLEAR(gen->exc_traceback); + __Pyx_Coroutine_ExceptionClear(&gen->gi_exc_state); +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + Py_CLEAR(((__pyx_PyAsyncGenObject*)gen)->ag_finalizer); + } +#endif + Py_CLEAR(gen->gi_code); + Py_CLEAR(gen->gi_frame); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); + Py_CLEAR(gen->gi_modulename); return 0; } @@ -862,14 +1208,14 @@ if (gen->gi_weakreflist != NULL) PyObject_ClearWeakRefs(self); - if (gen->resume_label > 0) { - // Generator is paused, so we need to close + if (gen->resume_label >= 0) { + // Generator is paused or unstarted, so we need to close PyObject_GC_Track(self); -#if PY_VERSION_HEX >= 0x030400a1 - if (PyObject_CallFinalizerFromDealloc(self)) +#if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) #else Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) + if (unlikely(Py_REFCNT(self) > 0)) #endif { // resurrected. :( @@ -878,44 +1224,114 @@ PyObject_GC_UnTrack(self); } +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + /* We have to handle this case for asynchronous generators + right here, because this code has to be between UNTRACK + and GC_Del. */ + Py_CLEAR(((__pyx_PyAsyncGenObject*)self)->ag_finalizer); + } +#endif __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); + __Pyx_PyHeapTypeObject_GC_Del(gen); } static void __Pyx_Coroutine_del(PyObject *self) { - PyObject *res; PyObject *error_type, *error_value, *error_traceback; __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; __Pyx_PyThreadState_declare - if (gen->resume_label <= 0) - return ; + if (gen->resume_label < 0) { + // already terminated => nothing to clean up + return; + } + +#if !CYTHON_USE_TP_FINALIZE + // Temporarily resurrect the object. + assert(self->ob_refcnt == 0); + __Pyx_SET_REFCNT(self, 1); +#endif + + __Pyx_PyThreadState_assign + + // Save the current exception, if any. + __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); + +#ifdef __Pyx_AsyncGen_USED + if (__Pyx_AsyncGen_CheckExact(self)) { + __pyx_PyAsyncGenObject *agen = (__pyx_PyAsyncGenObject*)self; + PyObject *finalizer = agen->ag_finalizer; + if (finalizer && !agen->ag_closed) { + PyObject *res = __Pyx_PyObject_CallOneArg(finalizer, self); + if (unlikely(!res)) { + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + // Restore the saved exception. + __Pyx_ErrRestore(error_type, error_value, error_traceback); + return; + } + } +#endif + + if (unlikely(gen->resume_label == 0 && !error_value)) { +#ifdef __Pyx_Coroutine_USED +#ifdef __Pyx_Generator_USED + // only warn about (async) coroutines + if (!__Pyx_Generator_CheckExact(self)) +#endif + { + // untrack dead object as we are executing Python code (which might trigger GC) + PyObject_GC_UnTrack(self); +#if PY_MAJOR_VERSION >= 3 /* PY_VERSION_HEX >= 0x03030000*/ || defined(PyErr_WarnFormat) + if (unlikely(PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname) < 0)) + PyErr_WriteUnraisable(self); +#else + {PyObject *msg; + char *cmsg; + #if CYTHON_COMPILING_IN_PYPY + msg = NULL; + cmsg = (char*) "coroutine was never awaited"; + #else + char *cname; + PyObject *qualname; + qualname = gen->gi_qualname; + cname = PyString_AS_STRING(qualname); + msg = PyString_FromFormat("coroutine '%.50s' was never awaited", cname); -#if PY_VERSION_HEX < 0x030400a1 - // Temporarily resurrect the object. - assert(self->ob_refcnt == 0); - self->ob_refcnt = 1; + if (unlikely(!msg)) { + PyErr_Clear(); + cmsg = (char*) "coroutine was never awaited"; + } else { + cmsg = PyString_AS_STRING(msg); + } + #endif + if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, cmsg, 1) < 0)) + PyErr_WriteUnraisable(self); + Py_XDECREF(msg);} #endif - - // Save the current exception, if any. - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&error_type, &error_value, &error_traceback); - - res = __Pyx_Coroutine_Close(self); - - if (res == NULL) - PyErr_WriteUnraisable(self); - else - Py_DECREF(res); + PyObject_GC_Track(self); + } +#endif /*__Pyx_Coroutine_USED*/ + } else { + PyObject *res = __Pyx_Coroutine_Close(self); + if (unlikely(!res)) { + if (PyErr_Occurred()) + PyErr_WriteUnraisable(self); + } else { + Py_DECREF(res); + } + } // Restore the saved exception. __Pyx_ErrRestore(error_type, error_value, error_traceback); -#if PY_VERSION_HEX < 0x030400a1 +#if !CYTHON_USE_TP_FINALIZE // Undo the temporary resurrection; can't use DECREF here, it would // cause a recursive call. - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { + assert(Py_REFCNT(self) > 0); + if (likely(--self->ob_refcnt == 0)) { // this is the normal path out return; } @@ -923,12 +1339,12 @@ // close() resurrected it! Make it look like the original Py_DECREF // never happened. { - Py_ssize_t refcnt = self->ob_refcnt; + Py_ssize_t refcnt = Py_REFCNT(self); _Py_NewReference(self); - self->ob_refcnt = refcnt; + __Pyx_SET_REFCNT(self, refcnt); } #if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && + assert(PyType_IS_GC(Py_TYPE(self)) && _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); // If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so @@ -948,69 +1364,102 @@ } static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self) +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context) { - Py_INCREF(self->gi_name); - return self->gi_name; + PyObject *name = self->gi_name; + CYTHON_UNUSED_VAR(context); + // avoid NULL pointer dereference during garbage collection + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; } static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value) +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__name__ must be set to a string object"); return -1; } - tmp = self->gi_name; Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_name, value); return 0; } static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self) +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context) { - Py_INCREF(self->gi_qualname); - return self->gi_qualname; + PyObject *name = self->gi_qualname; + CYTHON_UNUSED_VAR(context); + // avoid NULL pointer dereference during garbage collection + if (unlikely(!name)) name = Py_None; + Py_INCREF(name); + return name; } static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value) +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__qualname__ must be set to a string object"); return -1; } - tmp = self->gi_qualname; Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); return 0; } +static PyObject * +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) +{ + PyObject *frame = self->gi_frame; + CYTHON_UNUSED_VAR(context); + if (!frame) { + if (unlikely(!self->gi_code)) { + // Avoid doing something stupid, e.g. during garbage collection. + Py_RETURN_NONE; + } + frame = (PyObject *) PyFrame_New( + PyThreadState_Get(), /*PyThreadState *tstate,*/ + (PyCodeObject*) self->gi_code, /*PyCodeObject *code,*/ + $moddict_cname, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (unlikely(!frame)) + return NULL; + // keep the frame cached once it's created + self->gi_frame = frame; + } + Py_INCREF(frame); + return frame; +} + static __pyx_CoroutineObject *__Pyx__Coroutine_New( - PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *closure, + PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, PyObject *name, PyObject *qualname, PyObject *module_name) { __pyx_CoroutineObject *gen = PyObject_GC_New(__pyx_CoroutineObject, type); - - if (gen == NULL) + if (unlikely(!gen)) return NULL; + return __Pyx__Coroutine_NewInit(gen, body, code, closure, name, qualname, module_name); +} +static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( + __pyx_CoroutineObject *gen, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, + PyObject *name, PyObject *qualname, PyObject *module_name) { gen->body = body; gen->closure = closure; Py_XINCREF(closure); @@ -1018,9 +1467,16 @@ gen->resume_label = 0; gen->classobj = NULL; gen->yieldfrom = NULL; - gen->exc_type = NULL; - gen->exc_value = NULL; - gen->exc_traceback = NULL; + #if PY_VERSION_HEX >= 0x030B00a4 + gen->gi_exc_state.exc_value = NULL; + #else + gen->gi_exc_state.exc_type = NULL; + gen->gi_exc_state.exc_value = NULL; + gen->gi_exc_state.exc_traceback = NULL; + #endif +#if CYTHON_USE_EXC_INFO_STACK + gen->gi_exc_state.previous_item = NULL; +#endif gen->gi_weakreflist = NULL; Py_XINCREF(qualname); gen->gi_qualname = qualname; @@ -1028,6 +1484,9 @@ gen->gi_name = name; Py_XINCREF(module_name); gen->gi_modulename = module_name; + Py_XINCREF(code); + gen->gi_code = code; + gen->gi_frame = NULL; PyObject_GC_Track(gen); return gen; @@ -1037,16 +1496,12 @@ //////////////////// Coroutine //////////////////// //@requires: CoroutineBase //@requires: PatchGeneratorABC - -typedef struct { - PyObject_HEAD - PyObject *coroutine; -} __pyx_CoroutineAwaitObject; +//@requires: ObjectHandling.c::PyObject_GenericGetAttrNoDict static void __Pyx_CoroutineAwait_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(((__pyx_CoroutineAwaitObject*)self)->coroutine); - PyObject_GC_Del(self); + __Pyx_PyHeapTypeObject_GC_Del(self); } static int __Pyx_CoroutineAwait_traverse(__pyx_CoroutineAwaitObject *self, visitproc visit, void *arg) { @@ -1071,7 +1526,8 @@ return __Pyx_Coroutine_Throw(self->coroutine, args); } -static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self) { +static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_Coroutine_Close(self->coroutine); } @@ -1081,12 +1537,31 @@ } #if !CYTHON_COMPILING_IN_PYPY -static PyObject *__Pyx_CoroutineAwait_no_new(CYTHON_UNUSED PyTypeObject *type, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwargs) { +static PyObject *__Pyx_CoroutineAwait_no_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(args); + CYTHON_UNUSED_VAR(kwargs); PyErr_SetString(PyExc_TypeError, "cannot instantiate type, use 'await coroutine' instead"); return NULL; } #endif +// In earlier versions of Python an object with no __dict__ and not __slots__ is assumed +// to be pickleable by default. Coroutine-wrappers have significant state so shouldn't be. +// Therefore provide a default implementation. +// Something similar applies to heaptypes (i.e. with type_specs) with protocols 0 and 1 +// even in more recent versions. +// We are applying this to all Python versions (hence the commented out version guard) +// to make the behaviour explicit. +// #if PY_VERSION_HEX < 0x03060000 || CYTHON_USE_TYPE_SPECS +static PyObject *__Pyx_CoroutineAwait_reduce_ex(__pyx_CoroutineAwaitObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + PyErr_Format(PyExc_TypeError, "cannot pickle '%.200s' object", + Py_TYPE(self)->tp_name); + return NULL; +} +// #endif + static PyMethodDef __pyx_CoroutineAwait_methods[] = { {"send", (PyCFunction) __Pyx_CoroutineAwait_Send, METH_O, (char*) PyDoc_STR("send(arg) -> send 'arg' into coroutine,\nreturn next yielded value or raise StopIteration.")}, @@ -1094,12 +1569,40 @@ (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in coroutine,\nreturn next yielded value or raise StopIteration.")}, {"close", (PyCFunction) __Pyx_CoroutineAwait_Close, METH_NOARGS, (char*) PyDoc_STR("close() -> raise GeneratorExit inside coroutine.")}, +// only needed with type-specs or version<3.6, but included in all versions for clarity +// #if PY_VERSION_HEX < 0x03060000 || CYTHON_USE_TYPE_SPECS + {"__reduce_ex__", (PyCFunction) __Pyx_CoroutineAwait_reduce_ex, METH_O, 0}, + {"__reduce__", (PyCFunction) __Pyx_CoroutineAwait_reduce_ex, METH_NOARGS, 0}, +// #endif {0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineAwaitType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CoroutineAwait_dealloc}, + {Py_tp_traverse, (void *)__Pyx_CoroutineAwait_traverse}, + {Py_tp_clear, (void *)__Pyx_CoroutineAwait_clear}, +#if !CYTHON_COMPILING_IN_PYPY + {Py_tp_new, (void *)__Pyx_CoroutineAwait_no_new}, +#endif + {Py_tp_methods, (void *)__pyx_CoroutineAwait_methods}, + {Py_tp_iter, (void *)__Pyx_CoroutineAwait_self}, + {Py_tp_iternext, (void *)__Pyx_CoroutineAwait_Next}, + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineAwaitType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", + sizeof(__pyx_CoroutineAwaitObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx_CoroutineAwaitType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CoroutineAwaitType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine_wrapper", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", /*tp_name*/ sizeof(__pyx_CoroutineAwaitObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CoroutineAwait_dealloc,/*tp_dealloc*/ @@ -1152,90 +1655,45 @@ #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ +#if PY_VERSION_HEX < 0x030500B1 || defined(__Pyx_IterableCoroutine_USED) || CYTHON_USE_ASYNC_SLOTS static CYTHON_INLINE PyObject *__Pyx__Coroutine_await(PyObject *coroutine) { -#if CYTHON_COMPILING_IN_CPYTHON __pyx_CoroutineAwaitObject *await = PyObject_GC_New(__pyx_CoroutineAwaitObject, __pyx_CoroutineAwaitType); -#else - __pyx_CoroutineAwaitObject *await = (__pyx_CoroutineAwaitObject*) - __pyx_CoroutineAwaitType->tp_new(__pyx_CoroutineAwaitType, __pyx_empty_tuple, NULL); -#endif if (unlikely(!await)) return NULL; Py_INCREF(coroutine); await->coroutine = coroutine; -#if CYTHON_COMPILING_IN_CPYTHON PyObject_GC_Track(await); -#endif return (PyObject*)await; } +#endif + +#if PY_VERSION_HEX < 0x030500B1 +static PyObject *__Pyx_Coroutine_await_method(PyObject *coroutine, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + return __Pyx__Coroutine_await(coroutine); +} +#endif +#if defined(__Pyx_IterableCoroutine_USED) || CYTHON_USE_ASYNC_SLOTS static PyObject *__Pyx_Coroutine_await(PyObject *coroutine) { - if (unlikely(!coroutine || !__Pyx_Coroutine_CheckExact(coroutine))) { + if (unlikely(!coroutine || !__Pyx_Coroutine_Check(coroutine))) { PyErr_SetString(PyExc_TypeError, "invalid input, expected coroutine"); return NULL; } return __Pyx__Coroutine_await(coroutine); } - -static void __Pyx_Coroutine_check_and_dealloc(PyObject *self) { - __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; - - if (gen->resume_label == 0 && !PyErr_Occurred()) { - // untrack dead object as we are executing Python code (which might trigger GC) - PyObject_GC_UnTrack(self); -#if PY_VERSION_HEX >= 0x03030000 || defined(PyErr_WarnFormat) - PyErr_WarnFormat(PyExc_RuntimeWarning, 1, "coroutine '%.50S' was never awaited", gen->gi_qualname); - PyErr_Clear(); /* just in case, must not keep a live exception during GC */ -#else - {PyObject *msg; - char *cmsg; - #if CYTHON_COMPILING_IN_PYPY - msg = NULL; - cmsg = (char*) "coroutine was never awaited"; - #else - char *cname; - PyObject *qualname; - #if PY_MAJOR_VERSION >= 3 - qualname = PyUnicode_AsUTF8String(gen->gi_qualname); - if (likely(qualname)) { - cname = PyBytes_AS_STRING(qualname); - } else { - PyErr_Clear(); - cname = (char*) "?"; - } - msg = PyBytes_FromFormat( - #else - qualname = gen->gi_qualname; - cname = PyString_AS_STRING(qualname); - msg = PyString_FromFormat( - #endif - "coroutine '%.50s' was never awaited", cname); - - #if PY_MAJOR_VERSION >= 3 - Py_XDECREF(qualname); - #endif - - if (unlikely(!msg)) { - PyErr_Clear(); - cmsg = (char*) "coroutine was never awaited"; - } else { - #if PY_MAJOR_VERSION >= 3 - cmsg = PyBytes_AS_STRING(msg); - #else - cmsg = PyString_AS_STRING(msg); - #endif - } - #endif - if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, cmsg, 1) < 0)) - PyErr_WriteUnraisable(self); - Py_XDECREF(msg);} #endif - PyObject_GC_Track(self); - } - - __Pyx_Coroutine_dealloc(self); -} #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 static PyObject *__Pyx_Coroutine_compare(PyObject *obj, PyObject *other, int op) { @@ -1256,10 +1714,10 @@ (char*) PyDoc_STR("send(arg) -> send 'arg' into coroutine,\nreturn next iterated value or raise StopIteration.")}, {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in coroutine,\nreturn next iterated value or raise StopIteration.")}, - {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, + {"close", (PyCFunction) __Pyx_Coroutine_Close_Method, METH_NOARGS, (char*) PyDoc_STR("close() -> raise GeneratorExit inside coroutine.")}, #if PY_VERSION_HEX < 0x030500B1 - {"__await__", (PyCFunction) __Pyx_Coroutine_await, METH_NOARGS, + {"__await__", (PyCFunction) __Pyx_Coroutine_await_method, METH_NOARGS, (char*) PyDoc_STR("__await__() -> return an iterator to be used in await expression.")}, #endif {0, 0, 0, 0} @@ -1269,7 +1727,11 @@ {(char *) "cr_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, {(char*) "cr_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being awaited, or None")}, - {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), PY_WRITE_RESTRICTED, 0}, + {(char*) "cr_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1278,23 +1740,52 @@ (char*) PyDoc_STR("name of the coroutine"), 0}, {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, (char*) PyDoc_STR("qualified name of the coroutine"), 0}, + {(char *) "cr_frame", (getter)__Pyx_Coroutine_get_frame, NULL, + (char*) PyDoc_STR("Frame of the coroutine"), 0}, {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_CoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { __Pyx_Coroutine_await, /*am_await*/ 0, /*am_aiter*/ 0, /*am_anext*/ +#if PY_VERSION_HEX >= 0x030A00A3 + 0, /*am_send*/ +#endif }; #endif static PyTypeObject __pyx_CoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ - (destructor) __Pyx_Coroutine_check_and_dealloc,/*tp_dealloc*/ + (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -1317,14 +1808,14 @@ 0, /*tp_doc*/ (traverseproc) __Pyx_Coroutine_traverse, /*tp_traverse*/ 0, /*tp_clear*/ -#if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 +#if CYTHON_USE_ASYNC_SLOTS && CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 // in order to (mis-)use tp_reserved above, we must also implement tp_richcompare __Pyx_Coroutine_compare, /*tp_richcompare*/ #else 0, /*tp_richcompare*/ #endif offsetof(__pyx_CoroutineObject, gi_weakreflist), /*tp_weaklistoffset*/ -// no tp_iter() as iterator is only available through __await__() + // no tp_iter() as iterator is only available through __await__() 0, /*tp_iter*/ 0, /*tp_iternext*/ __pyx_Coroutine_methods, /*tp_methods*/ @@ -1345,41 +1836,205 @@ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE 0, /*tp_del*/ #else __Pyx_Coroutine_del, /*tp_del*/ #endif 0, /*tp_version_tag*/ -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE __Pyx_Coroutine_del, /*tp_finalize*/ +#elif PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Coroutine_init(void) { +static int __pyx_Coroutine_init(PyObject *module) { // on Windows, C-API functions can't be used in slots statically - __pyx_CoroutineType_type.tp_getattro = PyObject_GenericGetAttr; - +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineType_spec, NULL); +#else + (void) module; + __pyx_CoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_CoroutineType = __Pyx_FetchCommonType(&__pyx_CoroutineType_type); +#endif if (unlikely(!__pyx_CoroutineType)) return -1; +#ifdef __Pyx_IterableCoroutine_USED + if (unlikely(__pyx_IterableCoroutine_init(module) == -1)) + return -1; +#endif + +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineAwaitType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineAwaitType_spec, NULL); +#else __pyx_CoroutineAwaitType = __Pyx_FetchCommonType(&__pyx_CoroutineAwaitType_type); +#endif if (unlikely(!__pyx_CoroutineAwaitType)) return -1; return 0; } + +//////////////////// IterableCoroutine.proto //////////////////// + +#define __Pyx_IterableCoroutine_USED + +static PyTypeObject *__pyx_IterableCoroutineType = 0; + +#undef __Pyx_Coroutine_Check +#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || __Pyx_IS_TYPE(obj, __pyx_IterableCoroutineType)) + +#define __Pyx_IterableCoroutine_New(body, code, closure, name, qualname, module_name) \ + __Pyx__Coroutine_New(__pyx_IterableCoroutineType, body, code, closure, name, qualname, module_name) + +static int __pyx_IterableCoroutine_init(PyObject *module);/*proto*/ + + +//////////////////// IterableCoroutine //////////////////// +//@requires: Coroutine +//@requires: CommonStructures.c::FetchCommonType + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_IterableCoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)__Pyx_Coroutine_await}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_IterableCoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_IterableCoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + +static PyTypeObject __pyx_IterableCoroutineType_type = { + PyVarObject_HEAD_INIT(0, 0) + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", /*tp_name*/ + sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ +#if CYTHON_USE_ASYNC_SLOTS + &__pyx_Coroutine_as_async, /*tp_as_async (tp_reserved) - Py3 only! */ +#else + 0, /*tp_reserved*/ +#endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + 0, /*tp_doc*/ + (traverseproc) __Pyx_Coroutine_traverse, /*tp_traverse*/ + 0, /*tp_clear*/ +#if CYTHON_USE_ASYNC_SLOTS && CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 + // in order to (mis-)use tp_reserved above, we must also implement tp_richcompare + __Pyx_Coroutine_compare, /*tp_richcompare*/ +#else + 0, /*tp_richcompare*/ +#endif + offsetof(__pyx_CoroutineObject, gi_weakreflist), /*tp_weaklistoffset*/ + // enable iteration for legacy support of asyncio yield-from protocol + __Pyx_Coroutine_await, /*tp_iter*/ + (iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/ + __pyx_Coroutine_methods, /*tp_methods*/ + __pyx_Coroutine_memberlist, /*tp_members*/ + __pyx_Coroutine_getsets, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + 0, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ +#if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_del*/ +#else + __Pyx_Coroutine_del, /*tp_del*/ +#endif + 0, /*tp_version_tag*/ +#if PY_VERSION_HEX >= 0x030400a1 && !CYTHON_COMPILING_IN_PYPY + __Pyx_Coroutine_del, /*tp_finalize*/ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif +}; +#endif /* CYTHON_USE_TYPE_SPECS */ + + +static int __pyx_IterableCoroutine_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_IterableCoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_IterableCoroutineType_spec, NULL); +#else + (void) module; + __pyx_IterableCoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + __pyx_IterableCoroutineType = __Pyx_FetchCommonType(&__pyx_IterableCoroutineType_type); +#endif + if (unlikely(!__pyx_IterableCoroutineType)) + return -1; + return 0; +} + + //////////////////// Generator //////////////////// //@requires: CoroutineBase //@requires: PatchGeneratorABC +//@requires: ObjectHandling.c::PyObject_GenericGetAttrNoDict static PyMethodDef __pyx_Generator_methods[] = { {"send", (PyCFunction) __Pyx_Coroutine_Send, METH_O, (char*) PyDoc_STR("send(arg) -> send 'arg' into generator,\nreturn next yielded value or raise StopIteration.")}, {"throw", (PyCFunction) __Pyx_Coroutine_Throw, METH_VARARGS, (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in generator,\nreturn next yielded value or raise StopIteration.")}, - {"close", (PyCFunction) __Pyx_Coroutine_Close, METH_NOARGS, + {"close", (PyCFunction) __Pyx_Coroutine_Close_Method, METH_NOARGS, (char*) PyDoc_STR("close() -> raise GeneratorExit inside generator.")}, {0, 0, 0, 0} }; @@ -1388,6 +2043,11 @@ {(char *) "gi_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, + {(char*) "gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1396,19 +2056,46 @@ (char*) PyDoc_STR("name of the generator"), 0}, {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, (char*) PyDoc_STR("qualified name of the generator"), 0}, + {(char *) "gi_frame", (getter)__Pyx_Coroutine_get_frame, NULL, + (char*) PyDoc_STR("Frame of the generator"), 0}, {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_GeneratorType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Generator_methods}, + {Py_tp_members, (void *)__pyx_Generator_memberlist}, + {Py_tp_getset, (void *)__pyx_Generator_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_GeneratorType_spec = { + __PYX_TYPE_MODULE_PREFIX "generator", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_GeneratorType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_GeneratorType_type = { PyVarObject_HEAD_INIT(0, 0) - "generator", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "generator", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_compare / tp_as_async*/ + 0, /*tp_as_async*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -1445,23 +2132,39 @@ 0, /*tp_cache*/ 0, /*tp_subclasses*/ 0, /*tp_weaklist*/ -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE 0, /*tp_del*/ #else __Pyx_Coroutine_del, /*tp_del*/ #endif 0, /*tp_version_tag*/ -#if PY_VERSION_HEX >= 0x030400a1 +#if CYTHON_USE_TP_FINALIZE __Pyx_Coroutine_del, /*tp_finalize*/ +#elif PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ +#endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Generator_init(void) { +static int __pyx_Generator_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_GeneratorType_spec, NULL); +#else + (void) module; // on Windows, C-API functions can't be used in slots statically - __pyx_GeneratorType_type.tp_getattro = PyObject_GenericGetAttr; + __pyx_GeneratorType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); +#endif if (unlikely(!__pyx_GeneratorType)) { return -1; } @@ -1483,13 +2186,15 @@ // 1) Instantiating an exception just to pass back a value is costly. // 2) CPython 3.3 <= x < 3.5b1 crash in yield-from when the StopIteration is not instantiated. // 3) Passing a tuple as value into PyErr_SetObject() passes its items on as arguments. -// 4) If there is currently an exception being handled, we need to chain it. +// 4) Passing an exception as value will interpret it as an exception on unpacking and raise it (or unpack its value). +// 5) If there is currently an exception being handled, we need to chain it. static void __Pyx__ReturnWithStopIteration(PyObject* value) { PyObject *exc, *args; -#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_CPYTHON __Pyx_PyThreadState_declare - if ((PY_VERSION_HEX >= 0x03030000 && PY_VERSION_HEX < 0x030500B1) || unlikely(PyTuple_Check(value))) { + if ((PY_VERSION_HEX >= 0x03030000 && PY_VERSION_HEX < 0x030500B1) + || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) { args = PyTuple_New(1); if (unlikely(!args)) return; Py_INCREF(value); @@ -1502,13 +2207,20 @@ Py_INCREF(value); exc = value; } + #if CYTHON_FAST_THREAD_STATE __Pyx_PyThreadState_assign - if (!$local_tstate_cname->exc_type) { + #if CYTHON_USE_EXC_INFO_STACK + if (!$local_tstate_cname->exc_info->exc_value) + #else + if (!$local_tstate_cname->exc_type) + #endif + { // no chaining needed => avoid the overhead in PyErr_SetObject() Py_INCREF(PyExc_StopIteration); __Pyx_ErrRestore(PyExc_StopIteration, exc, NULL); return; } + #endif #else args = PyTuple_Pack(1, value); if (unlikely(!args)) return; @@ -1579,6 +2291,10 @@ //////////////////// PatchGeneratorABC //////////////////// //@requires: PatchModuleWithCoroutine +#ifndef CYTHON_REGISTER_ABCS +#define CYTHON_REGISTER_ABCS 1 +#endif + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) static PyObject* __Pyx_patch_abc_module(PyObject *module); /*proto*/ static PyObject* __Pyx_patch_abc_module(PyObject *module) { @@ -1601,13 +2317,13 @@ static int __Pyx_patch_abc(void) { #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) static int abc_patched = 0; - if (!abc_patched) { + if (CYTHON_REGISTER_ABCS && !abc_patched) { PyObject *module; - module = PyImport_ImportModule((PY_VERSION_HEX >= 0x03030000) ? "collections.abc" : "collections"); - if (!module) { + module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections"); + if (unlikely(!module)) { PyErr_WriteUnraisable(NULL); if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, - ((PY_VERSION_HEX >= 0x03030000) ? + ((PY_MAJOR_VERSION >= 3) ? "Cython module failed to register with collections.abc module" : "Cython module failed to register with collections module"), 1) < 0)) { return -1; @@ -1631,7 +2347,7 @@ } #else // avoid "unused" warning for __Pyx_Coroutine_patch_module() - if (0) __Pyx_Coroutine_patch_module(NULL, NULL); + if ((0)) __Pyx_Coroutine_patch_module(NULL, NULL); #endif return 0; } @@ -1672,7 +2388,8 @@ ); } else { PyErr_Clear(); -#if PY_VERSION_HEX < 0x03040200 +// Always enable fallback: even if we compile against 3.4.2, we might be running on 3.4.1 at some point. +//#if PY_VERSION_HEX < 0x03040200 // Py3.4.1 used to have asyncio.tasks instead of asyncio.coroutines package = __Pyx_Import(PYIDENT("asyncio.tasks"), NULL, 0); if (unlikely(!package)) goto asyncio_done; @@ -1693,15 +2410,15 @@ old_types.add(_cython_generator_type) """) ); -#endif +//#endif // Py < 0x03040200 } Py_DECREF(package); if (unlikely(!patch_module)) goto ignore; -#if PY_VERSION_HEX < 0x03040200 +//#if PY_VERSION_HEX < 0x03040200 asyncio_done: PyErr_Clear(); -#endif +//#endif asyncio_patched = 1; #ifdef __Pyx_Generator_USED // now patch inspect.isgenerator() by looking up the imported module in the patched asyncio module @@ -1723,7 +2440,7 @@ } #else // avoid "unused" warning for __Pyx_patch_inspect() - if (0) return __Pyx_patch_inspect(module); + if ((0)) return __Pyx_patch_inspect(module); #endif } return module; @@ -1735,7 +2452,7 @@ } #else // avoid "unused" warning for __Pyx_Coroutine_patch_module() - if (0) return __Pyx_patch_inspect(__Pyx_Coroutine_patch_module(module, NULL)); + if ((0)) return __Pyx_patch_inspect(__Pyx_Coroutine_patch_module(module, NULL)); #endif return module; } @@ -1770,7 +2487,7 @@ } #else // avoid "unused" warning for __Pyx_Coroutine_patch_module() - if (0) return __Pyx_Coroutine_patch_module(module, NULL); + if ((0)) return __Pyx_Coroutine_patch_module(module, NULL); #endif return module; } @@ -1780,11 +2497,15 @@ #define __Pyx_StopAsyncIteration_USED static PyObject *__Pyx_PyExc_StopAsyncIteration; -static int __pyx_StopAsyncIteration_init(void); /*proto*/ +static int __pyx_StopAsyncIteration_init(PyObject *module); /*proto*/ //////////////////// StopAsyncIteration //////////////////// #if PY_VERSION_HEX < 0x030500B1 +#if CYTHON_USE_TYPE_SPECS +#error Using async coroutines with type specs requires Python 3.5 or later. +#else + static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { PyVarObject_HEAD_INIT(0, 0) "StopAsyncIteration", /*tp_name*/ @@ -1836,11 +2557,16 @@ #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; #endif +#endif -static int __pyx_StopAsyncIteration_init(void) { +static int __pyx_StopAsyncIteration_init(PyObject *module) { #if PY_VERSION_HEX >= 0x030500B1 + (void) module; __Pyx_PyExc_StopAsyncIteration = PyExc_StopAsyncIteration; #else PyObject *builtins = PyEval_GetBuiltins(); @@ -1858,10 +2584,11 @@ __Pyx__PyExc_StopAsyncIteration_type.tp_dictoffset = ((PyTypeObject*)PyExc_BaseException)->tp_dictoffset; __Pyx__PyExc_StopAsyncIteration_type.tp_base = (PyTypeObject*)PyExc_Exception; + (void) module; __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type); if (unlikely(!__Pyx_PyExc_StopAsyncIteration)) return -1; - if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) + if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) return -1; #endif return 0; diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CpdefEnums.pyx cython-0.20.1+1~202203241016-9537/Cython/Utility/CpdefEnums.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CpdefEnums.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CpdefEnums.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -6,10 +6,7 @@ int PY_VERSION_HEX cdef object __Pyx_OrderedDict -if PY_VERSION_HEX >= 0x02070000: - from collections import OrderedDict as __Pyx_OrderedDict -else: - __Pyx_OrderedDict = dict +from collections import OrderedDict as __Pyx_OrderedDict @cython.internal cdef class __Pyx_EnumMeta(type): @@ -23,8 +20,7 @@ # @cython.internal cdef object __Pyx_EnumBase -class __Pyx_EnumBase(int): - __metaclass__ = __Pyx_EnumMeta +class __Pyx_EnumBase(int, metaclass=__Pyx_EnumMeta): def __new__(cls, value, name=None): for v in cls: if v == value: @@ -55,12 +51,38 @@ ('{{item}}', {{item}}), {{endfor}} ])) + {{if enum_doc is not None}} + {{name}}.__doc__ = {{ repr(enum_doc) }} + {{endif}} + {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}.{{item}} {{endfor}} else: class {{name}}(__Pyx_EnumBase): - pass + {{ repr(enum_doc) if enum_doc is not None else 'pass' }} {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}({{item}}, '{{item}}') {{endfor}} + +#################### CppScopedEnumType #################### +#@requires: EnumBase +cdef dict __Pyx_globals = globals() + +if PY_VERSION_HEX >= 0x03040000: + # create new IntEnum() + __Pyx_globals["{{name}}"] = __Pyx_EnumBase('{{name}}', __Pyx_OrderedDict([ + {{for item in items}} + ('{{item}}', <{{underlying_type}}>({{name}}.{{item}})), + {{endfor}} + ])) + +else: + __Pyx_globals["{{name}}"] = type('{{name}}', (__Pyx_EnumBase,), {}) + {{for item in items}} + __Pyx_globals["{{name}}"](<{{underlying_type}}>({{name}}.{{item}}), '{{item}}') + {{endfor}} + +{{if enum_doc is not None}} +__Pyx_globals["{{name}}"].__doc__ = {{ repr(enum_doc) }} +{{endif}} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CppConvert.pyx cython-0.20.1+1~202203241016-9537/Cython/Utility/CppConvert.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CppConvert.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CppConvert.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -5,14 +5,14 @@ cdef extern from *: cdef cppclass string "{{type}}": - string() - string(char* c_str, size_t size) - cdef char* __Pyx_PyObject_AsStringAndSize(object, Py_ssize_t*) except NULL + string() except + + string(char* c_str, size_t size) except + + cdef const char* __Pyx_PyObject_AsStringAndSize(object, Py_ssize_t*) except NULL @cname("{{cname}}") cdef string {{cname}}(object o) except *: - cdef Py_ssize_t length - cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + cdef Py_ssize_t length = 0 + cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) return string(data, length) @@ -27,7 +27,7 @@ {{for py_type in ['PyObject', 'PyUnicode', 'PyStr', 'PyBytes', 'PyByteArray']}} cdef extern from *: - cdef object __Pyx_{{py_type}}_FromStringAndSize(char*, size_t) + cdef object __Pyx_{{py_type}}_FromStringAndSize(const char*, size_t) @cname("{{cname.replace("PyObject", py_type, 1)}}") cdef inline object {{cname.replace("PyObject", py_type, 1)}}(const string& s): @@ -37,47 +37,59 @@ #################### vector.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass vector "std::vector" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef vector[X] {{cname}}(object o) except *: cdef vector[X] v for item in o: - v.push_back(X_from_py(item)) + v.push_back(item) return v #################### vector.to_py #################### -{{template_type_declarations}} - cdef extern from *: - cdef cppclass vector "const std::vector" [T]: + cdef cppclass vector "std::vector" [T]: size_t size() T& operator[](size_t) -@cname("{{cname}}") -cdef object {{cname}}(vector[X]& v): - return [X_to_py(v[i]) for i in range(v.size())] +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX + +@cname("{{cname}}") +cdef object {{cname}}(const vector[X]& v): + if v.size() > PY_SSIZE_T_MAX: + raise MemoryError() + + o = PyList_New( v.size()) + + cdef Py_ssize_t i + cdef object item + + for i in range(v.size()): + item = v[i] + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) + return o #################### list.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass cpp_list "std::list" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef cpp_list[X] {{cname}}(object o) except *: cdef cpp_list[X] l for item in o: - l.push_back(X_from_py(item)) + l.push_back(item) return l @@ -85,8 +97,6 @@ cimport cython -{{template_type_declarations}} - cdef extern from *: cdef cppclass cpp_list "std::list" [T]: cppclass const_iterator: @@ -95,30 +105,46 @@ bint operator!=(const_iterator) const_iterator begin() const_iterator end() + size_t size() + +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX @cname("{{cname}}") cdef object {{cname}}(const cpp_list[X]& v): - o = [] + if v.size() > PY_SSIZE_T_MAX: + raise MemoryError() + + o = PyList_New( v.size()) + + cdef object item + cdef Py_ssize_t i = 0 cdef cpp_list[X].const_iterator iter = v.begin() + while iter != v.end(): - o.append(X_to_py(cython.operator.dereference(iter))) + item = cython.operator.dereference(iter) + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) cython.operator.preincrement(iter) + i += 1 + return o #################### set.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass set "std::{{maybe_unordered}}set" [T]: - void insert(T&) + void insert(T&) except + @cname("{{cname}}") cdef set[X] {{cname}}(object o) except *: cdef set[X] s for item in o: - s.insert(X_from_py(item)) + s.insert(item) return s @@ -126,8 +152,6 @@ cimport cython -{{template_type_declarations}} - cdef extern from *: cdef cppclass cpp_set "std::{{maybe_unordered}}set" [T]: cppclass const_iterator: @@ -139,32 +163,23 @@ @cname("{{cname}}") cdef object {{cname}}(const cpp_set[X]& s): - o = set() - cdef cpp_set[X].const_iterator iter = s.begin() - while iter != s.end(): - o.add(X_to_py(cython.operator.dereference(iter))) - cython.operator.preincrement(iter) - return o + return {v for v in s} #################### pair.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair() - pair(T&, U&) + pair() except + + pair(T&, U&) except + @cname("{{cname}}") cdef pair[X,Y] {{cname}}(object o) except *: x, y = o - return pair[X,Y](X_from_py(x), Y_from_py(y)) + return pair[X,Y](x, y) #################### pair.to_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass pair "std::pair" [T, U]: T first @@ -172,28 +187,30 @@ @cname("{{cname}}") cdef object {{cname}}(const pair[X,Y]& p): - return X_to_py(p.first), Y_to_py(p.second) + return p.first, p.second #################### map.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair(T&, U&) + pair(T&, U&) except + cdef cppclass map "std::{{maybe_unordered}}map" [T, U]: - void insert(pair[T, U]&) + void insert(pair[T, U]&) except + cdef cppclass vector "std::vector" [T]: pass + int PY_MAJOR_VERSION @cname("{{cname}}") cdef map[X,Y] {{cname}}(object o) except *: - cdef dict d = o cdef map[X,Y] m - for key, value in d.iteritems(): - m.insert(pair[X,Y](X_from_py(key), Y_from_py(value))) + if PY_MAJOR_VERSION < 3: + for key, value in o.iteritems(): + m.insert(pair[X,Y](key, value)) + else: + for key, value in o.items(): + m.insert(pair[X,Y](key, value)) return m @@ -203,8 +220,6 @@ cimport cython -{{template_type_declarations}} - cdef extern from *: cdef cppclass map "std::{{maybe_unordered}}map" [T, U]: cppclass value_type: @@ -224,15 +239,13 @@ cdef map[X,Y].const_iterator iter = s.begin() while iter != s.end(): key_value = &cython.operator.dereference(iter) - o[X_to_py(key_value.first)] = Y_to_py(key_value.second) + o[key_value.first] = key_value.second cython.operator.preincrement(iter) return o #################### complex.from_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass std_complex "std::complex" [T]: std_complex() @@ -246,8 +259,6 @@ #################### complex.to_py #################### -{{template_type_declarations}} - cdef extern from *: cdef cppclass std_complex "std::complex" [T]: X real() diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CppSupport.cpp cython-0.20.1+1~202203241016-9537/Cython/Utility/CppSupport.cpp --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CppSupport.cpp 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CppSupport.cpp 2022-03-24 10:16:46.000000000 +0000 @@ -46,3 +46,53 @@ } } #endif + +/////////////// PythranConversion.proto /////////////// + +template +auto __Pyx_pythran_to_python(T &&value) -> decltype(to_python( + typename pythonic::returnable::type>::type>::type{std::forward(value)})) +{ + using returnable_type = typename pythonic::returnable::type>::type>::type; + return to_python(returnable_type{std::forward(value)}); +} + +#define __Pyx_PythranShapeAccessor(x) (pythonic::builtins::getattr(pythonic::types::attr::SHAPE{}, x)) + +////////////// MoveIfSupported.proto ////////////////// + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include + #define __PYX_STD_MOVE_IF_SUPPORTED(x) std::move(x) +#else + #define __PYX_STD_MOVE_IF_SUPPORTED(x) x +#endif + +////////////// EnumClassDecl.proto ////////////////// + +#if defined (_MSC_VER) + #if _MSC_VER >= 1910 + #define __PYX_ENUM_CLASS_DECL enum + #else + #define __PYX_ENUM_CLASS_DECL + #endif +#else + #define __PYX_ENUM_CLASS_DECL enum +#endif + +////////////// OptionalLocals.proto //////////////// +//@proto_block: utility_code_proto_before_types + +#if defined(CYTHON_USE_BOOST_OPTIONAL) + // fallback mode - std::optional is preferred but this gives + // people with a less up-to-date compiler a chance + #include + #define __Pyx_Optional_Type boost::optional +#else + #include + // since std::optional is a C++17 features, a templated using declaration should be safe + // (although it could be replaced with a define) + template + using __Pyx_Optional_Type = std::optional; +#endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/CythonFunction.c cython-0.20.1+1~202203241016-9537/Cython/Utility/CythonFunction.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/CythonFunction.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/CythonFunction.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,17 +1,25 @@ +//////////////////// CythonFunctionShared.proto //////////////////// -//////////////////// CythonFunction.proto //////////////////// -#define __Pyx_CyFunction_USED 1 -#include +#define __Pyx_CyFunction_USED #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 #define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 #define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 #define __Pyx_CyFunction_GetClosure(f) \ (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f) \ - (((__pyx_CyFunctionObject *) (f))->func_classobj) + +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_CyFunction_GetClassObj(f) \ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f) \ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj) \ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) #define __Pyx_CyFunction_Defaults(type, f) \ ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) @@ -20,7 +28,15 @@ typedef struct { +#if PY_VERSION_HEX < 0x030900B1 PyCFunctionObject func; +#else + // PEP-573: PyCFunctionObject + mm_class + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL + __pyx_vectorcallfunc func_vectorcall; +#endif #if PY_VERSION_HEX < 0x030500A0 PyObject *func_weakreflist; #endif @@ -31,12 +47,14 @@ PyObject *func_globals; PyObject *func_code; PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 // No-args super() class cell PyObject *func_classobj; - +#endif // Dynamic default args and annotations void *defaults; int defaults_pyobjects; + size_t defaults_size; // used by FusedFunction for copying defaults int flags; // Defaults info @@ -44,19 +62,26 @@ PyObject *defaults_kwdict; /* Const kwonly defaults dict */ PyObject *(*defaults_getter)(PyObject *); PyObject *func_annotations; /* function annotations dict */ + + // Coroutine marker + PyObject *func_is_coroutine; } __pyx_CyFunctionObject; +#if !CYTHON_USE_MODULE_STATE static PyTypeObject *__pyx_CyFunctionType = 0; +#endif -#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code) \ - __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_CyFunctionType) +#define __Pyx_IsCyOrPyCFunction(obj) __Pyx_TypeCheck2(obj, __pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CyFunctionType) -static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *self, + PyObject *closure, PyObject *module, PyObject *globals, PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, size_t size, int pyobjects); @@ -68,22 +93,51 @@ PyObject *dict); -static int __pyx_CyFunction_init(void); +static int __pyx_CyFunction_init(PyObject *module); -//////////////////// CythonFunction //////////////////// +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +//////////////////// CythonFunctionShared //////////////////// //@substitute: naming -//@requires: CommonTypes.c::FetchCommonType -////@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: CommonStructures.c::FetchCommonType +//@requires: ObjectHandling.c::PyMethodNew +//@requires: ObjectHandling.c::PyVectorcallFastCallDict +//@requires: ModuleSetupCode.c::IncludeStructmemberH +//@requires: ObjectHandling.c::PyObjectGetAttrStr + +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + // assigning to "mm_class", which is a "PyTypeObject*" + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + CYTHON_UNUSED_VAR(closure); if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { #if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyString_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #endif if (unlikely(op->func_doc == NULL)) return NULL; @@ -97,27 +151,27 @@ } static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp = op->func_doc; + CYTHON_UNUSED_VAR(context); if (value == NULL) { // Mark as deleted value = Py_None; } Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_doc, value); return 0; } static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_name == NULL)) { #if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyString_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #endif if (unlikely(op->func_name == NULL)) return NULL; @@ -127,69 +181,55 @@ } static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_name, value); return 0; } static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_qualname); return op->func_qualname; } static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); return 0; } static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} - -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_dict == NULL)) { op->func_dict = PyDict_New(); if (unlikely(op->func_dict == NULL)) @@ -200,10 +240,9 @@ } static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); if (unlikely(value == NULL)) { PyErr_SetString(PyExc_TypeError, "function's dictionary may not be deleted"); @@ -214,31 +253,33 @@ "setting function's dictionary to a non-dict"); return -1; } - tmp = op->func_dict; Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_dict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_globals); return op->func_globals; } static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); Py_INCREF(Py_None); return Py_None; } static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) { PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); Py_INCREF(result); return result; } @@ -269,29 +310,30 @@ } static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { - PyObject* tmp; +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { PyErr_SetString(PyExc_TypeError, "__defaults__ must be set to a tuple object"); return -1; } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); return 0; } static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_tuple; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_tuple; } else { result = Py_None; @@ -302,29 +344,30 @@ } static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { - PyObject* tmp; +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__kwdefaults__ must be set to a dict object"); return -1; } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_kwdict; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_kwdict; } else { result = Py_None; @@ -335,25 +378,24 @@ } static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { - PyObject* tmp; +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value || value == Py_None) { value = NULL; - } else if (!PyDict_Check(value)) { + } else if (unlikely(!PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__annotations__ must be set to a dict object"); return -1; } Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); return 0; } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->func_annotations; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { result = PyDict_New(); if (unlikely(!result)) return NULL; @@ -363,10 +405,44 @@ return result; } +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + int is_coroutine; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + + is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; +#if PY_VERSION_HEX >= 0x03050000 + if (is_coroutine) { + PyObject *module, *fromlist, *marker = PYIDENT("_is_coroutine"); + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); + PyList_SET_ITEM(fromlist, 0, marker); + module = PyImport_ImportModuleLevelObject(PYIDENT("asyncio.coroutines"), NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + op->func_is_coroutine = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(op->func_is_coroutine)) { + return __Pyx_NewRef(op->func_is_coroutine); + } +ignore: + PyErr_Clear(); + } +#endif + + op->func_is_coroutine = __Pyx_PyBool_FromLong(is_coroutine); + return __Pyx_NewRef(op->func_is_coroutine); +} + //#if PY_VERSION_HEX >= 0x030400C1 //static PyObject * -//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op) { +//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, void *context) { // PyObject *inspect_module, *signature_class, *signature; +// CYTHON_UNUSED_VAR(context); // // from inspect import Signature // inspect_module = PyImport_ImportModuleLevelObject(PYIDENT("inspect"), NULL, NULL, NULL, 0); // if (unlikely(!inspect_module)) @@ -394,7 +470,6 @@ {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, @@ -407,6 +482,7 @@ {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {(char *) "_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, //#if PY_VERSION_HEX >= 0x030400C1 // {(char *) "__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, //#endif @@ -414,17 +490,34 @@ }; static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#endif +#if PY_VERSION_HEX < 0x030500A0 + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif {0, 0, 0, 0, 0} }; static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) { + CYTHON_UNUSED_VAR(args); #if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromString(m->func.m_ml->ml_name); + Py_INCREF(m->func_qualname); + return m->func_qualname; #else - return PyString_FromString(m->func.m_ml->ml_name); + return PyString_FromString(((PyCFunctionObject*)m)->m_ml->ml_name); #endif } @@ -437,41 +530,70 @@ #if PY_VERSION_HEX < 0x030500A0 #define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) #else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) #endif - -static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { - __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); - if (op == NULL) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyCFunctionObject *cf = (PyCFunctionObject*) op; + if (unlikely(op == NULL)) return NULL; op->flags = flags; __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; + cf->m_ml = ml; + cf->m_self = (PyObject *) op; Py_XINCREF(closure); op->func_closure = closure; Py_XINCREF(module); - op->func.m_module = module; + cf->m_module = module; op->func_dict = NULL; op->func_name = NULL; Py_INCREF(qualname); op->func_qualname = qualname; op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif op->func_globals = globals; Py_INCREF(op->func_globals); Py_XINCREF(code); op->func_code = code; // Dynamic Default args op->defaults_pyobjects = 0; + op->defaults_size = 0; op->defaults = NULL; op->defaults_tuple = NULL; op->defaults_kwdict = NULL; op->defaults_getter = NULL; op->func_annotations = NULL; - PyObject_GC_Track(op); + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + // case METH_FASTCALL is not used + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + // case METH_VARARGS is not used + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif return (PyObject *) op; } @@ -479,17 +601,26 @@ __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) { Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); + Py_CLEAR(((PyCFunctionObject*)m)->m_module); Py_CLEAR(m->func_dict); Py_CLEAR(m->func_name); Py_CLEAR(m->func_qualname); Py_CLEAR(m->func_doc); Py_CLEAR(m->func_globals); Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif Py_CLEAR(m->defaults_tuple); Py_CLEAR(m->defaults_kwdict); Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -505,28 +636,34 @@ return 0; } -static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) { - PyObject_GC_UnTrack(m); if (__Pyx_CyFunction_weakreflist(m) != NULL) PyObject_ClearWeakRefs((PyObject *) m); __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} + +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); } static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) { Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); + Py_VISIT(((PyCFunctionObject*)m)->m_module); Py_VISIT(m->func_dict); Py_VISIT(m->func_name); Py_VISIT(m->func_qualname); Py_VISIT(m->func_doc); Py_VISIT(m->func_globals); Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); Py_VISIT(m->defaults_tuple); Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -539,26 +676,6 @@ return 0; } -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - - if (obj == Py_None) - obj = NULL; - return __Pyx_PyMethod_New(func, obj, type); -} - static PyObject* __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) { @@ -583,7 +700,7 @@ return (*meth)(self, arg); break; case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); + return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); case METH_NOARGS: if (likely(kw == NULL || PyDict_Size(kw) == 0)) { size = PyTuple_GET_SIZE(arg); @@ -599,10 +716,16 @@ if (likely(kw == NULL || PyDict_Size(kw) == 0)) { size = PyTuple_GET_SIZE(arg); if (likely(size == 1)) { - PyObject *result, *arg0 = PySequence_ITEM(arg, 0); - if (unlikely(!arg0)) return NULL; + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(arg0); + #endif return result; } PyErr_Format(PyExc_TypeError, @@ -612,10 +735,7 @@ } break; default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); return NULL; } PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", @@ -630,6 +750,22 @@ static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { PyObject *result; __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + +#if CYTHON_METH_FASTCALL + // Prefer vectorcall if available. This is not the typical case, as + // CPython would normally use vectorcall directly instead of tp_call. + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + // avoid unused function warning + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { Py_ssize_t argc; PyObject *new_args; @@ -655,19 +791,198 @@ return result; } +#if CYTHON_METH_FASTCALL +// Check that kwnames is empty (if you want to allow keyword arguments, +// simply pass kwnames=NULL) and figure out what to do with "self". +// Return value: +// 1: self = args[0] +// 0: self = cyfunc->func.m_self +// -1: error +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + PyErr_Format(PyExc_TypeError, "%.200s() needs an argument", + ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no keyword arguments", ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + return ret; +} + +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 0)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, NULL); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 1)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, args[0]); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((_PyCFunctionFastWithKeywords)(void(*)(void))def->ml_meth)(self, args, nargs, kwnames); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((__Pyx_PyCMethod)(void(*)(void))def->ml_meth)(self, cls, args, nargs, kwnames); +} +#endif + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; + +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if (defined(_Py_TPFLAGS_HAVE_VECTORCALL) && CYTHON_METH_FASTCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_CyFunctionType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", /*tp_name*/ sizeof(__pyx_CyFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CyFunction_dealloc, /*tp_dealloc*/ +#if !CYTHON_METH_FASTCALL 0, /*tp_print*/ +#elif CYTHON_BACKPORT_VECTORCALL + (printfunc)offsetof(__pyx_CyFunctionObject, func_vectorcall), /*tp_vectorcall_offset backported into tp_print*/ +#else + offsetof(PyCFunctionObject, vectorcall), /*tp_vectorcall_offset*/ +#endif 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif (reprfunc) __Pyx_CyFunction_repr, /*tp_repr*/ 0, /*tp_as_number*/ @@ -679,7 +994,13 @@ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#ifdef _Py_TPFLAGS_HAVE_VECTORCALL + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ (traverseproc) __Pyx_CyFunction_traverse, /*tp_traverse*/ (inquiry) __Pyx_CyFunction_clear, /*tp_clear*/ @@ -696,7 +1017,7 @@ __pyx_CyFunction_getsets, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ - __Pyx_CyFunction_descr_get, /*tp_descr_get*/ + __Pyx_PyMethod_New, /*tp_descr_get*/ 0, /*tp_descr_set*/ offsetof(__pyx_CyFunctionObject, func_dict),/*tp_dictoffset*/ 0, /*tp_init*/ @@ -714,12 +1035,27 @@ #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_CyFunction_init(void) { +static int __pyx_CyFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CyFunctionType_spec, NULL); +#else + (void) module; __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); - if (__pyx_CyFunctionType == NULL) { +#endif + if (unlikely(__pyx_CyFunctionType == NULL)) { return -1; } return 0; @@ -729,10 +1065,11 @@ __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; m->defaults = PyObject_Malloc(size); - if (!m->defaults) + if (unlikely(!m->defaults)) return PyErr_NoMemory(); memset(m->defaults, 0, size); m->defaults_pyobjects = pyobjects; + m->defaults_size = size; return m->defaults; } @@ -754,13 +1091,38 @@ Py_INCREF(dict); } + +//////////////////// CythonFunction.proto //////////////////// + +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +//////////////////// CythonFunction //////////////////// +//@requires: CythonFunctionShared + +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + + //////////////////// CyFunctionClassCell.proto //////////////////// -static CYTHON_INLINE int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj); +static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj);/*proto*/ //////////////////// CyFunctionClassCell //////////////////// -//@requires: CythonFunction +//@requires: CythonFunctionShared -static CYTHON_INLINE int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj) { +static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *classobj) { Py_ssize_t i, count = PyList_GET_SIZE(cyfunctions); for (i = 0; i < count; i++) { @@ -772,8 +1134,7 @@ if (unlikely(!m)) return -1; #endif - Py_INCREF(classobj); - m->func_classobj = classobj; + __Pyx_CyFunction_SetClassObj(m, classobj); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF((PyObject*)m); #endif @@ -781,52 +1142,58 @@ return 0; } + //////////////////// FusedFunction.proto //////////////////// + typedef struct { __pyx_CyFunctionObject func; PyObject *__signatures__; - PyObject *type; PyObject *self; } __pyx_FusedFunctionObject; -#define __pyx_FusedFunction_NewEx(ml, flags, qualname, self, module, globals, code) \ - __pyx_FusedFunction_New(__pyx_FusedFunctionType, ml, flags, qualname, self, module, globals, code) -static PyObject *__pyx_FusedFunction_New(PyTypeObject *type, - PyMethodDef *ml, int flags, - PyObject *qualname, PyObject *self, +static PyObject *__pyx_FusedFunction_New(PyMethodDef *ml, int flags, + PyObject *qualname, PyObject *closure, PyObject *module, PyObject *globals, PyObject *code); static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self); +#if !CYTHON_USE_MODULE_STATE static PyTypeObject *__pyx_FusedFunctionType = NULL; -static int __pyx_FusedFunction_init(void); +#endif +static int __pyx_FusedFunction_init(PyObject *module); #define __Pyx_FusedFunction_USED //////////////////// FusedFunction //////////////////// -//@requires: CythonFunction +//@requires: CythonFunctionShared static PyObject * -__pyx_FusedFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, - PyObject *qualname, PyObject *self, +__pyx_FusedFunction_New(PyMethodDef *ml, int flags, + PyObject *qualname, PyObject *closure, PyObject *module, PyObject *globals, PyObject *code) { - __pyx_FusedFunctionObject *fusedfunc = - (__pyx_FusedFunctionObject *) __Pyx_CyFunction_New(type, ml, flags, qualname, - self, module, globals, code); - if (!fusedfunc) - return NULL; - - fusedfunc->__signatures__ = NULL; - fusedfunc->type = NULL; - fusedfunc->self = NULL; - return (PyObject *) fusedfunc; + PyObject *op = __Pyx_CyFunction_Init( + // __pyx_CyFunctionObject is correct below since that's the cast that we want. + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_FusedFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + __pyx_FusedFunctionObject *fusedfunc = (__pyx_FusedFunctionObject *) op; + fusedfunc->__signatures__ = NULL; + fusedfunc->self = NULL; + PyObject_GC_Track(op); + } + return op; } -static void __pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) { - __pyx_FusedFunction_clear(self); - __pyx_FusedFunctionType->tp_free((PyObject *) self); +static void +__pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(self->self); + Py_CLEAR(self->__signatures__); + __Pyx__CyFunction_dealloc((__pyx_CyFunctionObject *) self); } static int @@ -835,7 +1202,6 @@ void *arg) { Py_VISIT(self->self); - Py_VISIT(self->type); Py_VISIT(self->__signatures__); return __Pyx_CyFunction_traverse((__pyx_CyFunctionObject *) self, visit, arg); } @@ -844,7 +1210,6 @@ __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self) { Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); return __Pyx_CyFunction_clear((__pyx_CyFunctionObject *) self); } @@ -866,7 +1231,16 @@ if (obj == Py_None) obj = NULL; - meth = (__pyx_FusedFunctionObject *) __pyx_FusedFunction_NewEx( + if (func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD) + obj = type; + + if (obj == NULL) { + // We aren't actually binding to anything, save the effort of rebinding + Py_INCREF(self); + return self; + } + + meth = (__pyx_FusedFunctionObject *) __pyx_FusedFunction_New( ((PyCFunctionObject *) func)->m_ml, ((__pyx_CyFunctionObject *) func)->flags, ((__pyx_CyFunctionObject *) func)->func_qualname, @@ -874,24 +1248,38 @@ ((PyCFunctionObject *) func)->m_module, ((__pyx_CyFunctionObject *) func)->func_globals, ((__pyx_CyFunctionObject *) func)->func_code); - if (!meth) + if (unlikely(!meth)) return NULL; - Py_XINCREF(func->func.func_classobj); - meth->func.func_classobj = func->func.func_classobj; + // defaults needs copying fully rather than just copying the pointer + // since otherwise it will be freed on destruction of meth despite + // belonging to func rather than meth + if (func->func.defaults) { + PyObject **pydefaults; + int i; + + if (unlikely(!__Pyx_CyFunction_InitDefaults( + (PyObject*)meth, + func->func.defaults_size, + func->func.defaults_pyobjects))) { + Py_XDECREF((PyObject*)meth); + return NULL; + } + memcpy(meth->func.defaults, func->func.defaults, func->func.defaults_size); + + pydefaults = __Pyx_CyFunction_Defaults(PyObject *, meth); + for (i = 0; i < meth->func.defaults_pyobjects; i++) + Py_XINCREF(pydefaults[i]); + } + + __Pyx_CyFunction_SetClassObj(meth, __Pyx_CyFunction_GetClassObj(func)); Py_XINCREF(func->__signatures__); meth->__signatures__ = func->__signatures__; - Py_XINCREF(type); - meth->type = type; - Py_XINCREF(func->func.defaults_tuple); meth->func.defaults_tuple = func->func.defaults_tuple; - if (func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD) - obj = type; - Py_XINCREF(obj); meth->self = obj; @@ -899,12 +1287,18 @@ } static PyObject * -_obj_to_str(PyObject *obj) +_obj_to_string(PyObject *obj) { - if (PyType_Check(obj)) + if (PyUnicode_CheckExact(obj)) + return __Pyx_NewRef(obj); +#if PY_MAJOR_VERSION == 2 + else if (PyString_Check(obj)) + return PyUnicode_FromEncodedObject(obj, NULL, "strict"); +#endif + else if (PyType_Check(obj)) return PyObject_GetAttr(obj, PYIDENT("__name__")); else - return PyObject_Str(obj); + return PyObject_Unicode(obj); } static PyObject * @@ -914,64 +1308,55 @@ PyObject *unbound_result_func; PyObject *result_func = NULL; - if (self->__signatures__ == NULL) { + if (unlikely(self->__signatures__ == NULL)) { PyErr_SetString(PyExc_TypeError, "Function is not fused"); return NULL; } if (PyTuple_Check(idx)) { - PyObject *list = PyList_New(0); Py_ssize_t n = PyTuple_GET_SIZE(idx); - PyObject *string = NULL; - PyObject *sep = NULL; + PyObject *list = PyList_New(n); int i; - if (!list) + if (unlikely(!list)) return NULL; for (i = 0; i < n; i++) { + PyObject *string; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *item = PyTuple_GET_ITEM(idx, i); #else - PyObject *item = PySequence_ITEM(idx, i); + PyObject *item = PySequence_ITEM(idx, i); if (unlikely(!item)) goto __pyx_err; #endif - string = _obj_to_str(item); + string = _obj_to_string(item); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(item); #endif - if (!string || PyList_Append(list, string) < 0) - goto __pyx_err; - - Py_DECREF(string); + if (unlikely(!string)) goto __pyx_err; + PyList_SET_ITEM(list, i, string); } - sep = PyUnicode_FromString("|"); - if (sep) - signature = PyUnicode_Join(sep, list); -__pyx_err: -; + signature = PyUnicode_Join(PYUNICODE("|"), list); +__pyx_err:; Py_DECREF(list); - Py_XDECREF(sep); } else { - signature = _obj_to_str(idx); + signature = _obj_to_string(idx); } - if (!signature) + if (unlikely(!signature)) return NULL; unbound_result_func = PyObject_GetItem(self->__signatures__, signature); - if (unbound_result_func) { - if (self->self || self->type) { + if (likely(unbound_result_func)) { + if (self->self) { __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func; // TODO: move this to InitClassCell - Py_CLEAR(unbound->func.func_classobj); - Py_XINCREF(self->func.func_classobj); - unbound->func.func_classobj = self->func.func_classobj; + __Pyx_CyFunction_SetClassObj(unbound, __Pyx_CyFunction_GetClassObj(self)); result_func = __pyx_FusedFunction_descr_get(unbound_result_func, - self->self, self->type); + self->self, self->self); } else { result_func = unbound_result_func; Py_INCREF(result_func); @@ -991,7 +1376,7 @@ int static_specialized = (cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD && !((__pyx_FusedFunctionObject *) func)->__signatures__); - if (cyfunc->flags & __Pyx_CYFUNCTION_CCLASS && !static_specialized) { + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !static_specialized) { return __Pyx_CyFunction_CallAsMethod(func, args, kw); } else { return __Pyx_CyFunction_Call(func, args, kw); @@ -1012,23 +1397,21 @@ PyObject *new_args = NULL; __pyx_FusedFunctionObject *new_func = NULL; PyObject *result = NULL; - PyObject *self = NULL; int is_staticmethod = binding_func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD; - int is_classmethod = binding_func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD; if (binding_func->self) { // Bound method call, put 'self' in the args tuple + PyObject *self; Py_ssize_t i; new_args = PyTuple_New(argc + 1); - if (!new_args) + if (unlikely(!new_args)) return NULL; self = binding_func->self; -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_INCREF(self); -#endif + Py_INCREF(self); PyTuple_SET_ITEM(new_args, 0, self); + self = NULL; for (i = 0; i < argc; i++) { #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS @@ -1041,36 +1424,8 @@ } args = new_args; - } else if (binding_func->type) { - // Unbound method call - if (argc < 1) { - PyErr_SetString(PyExc_TypeError, "Need at least one argument, 0 given."); - return NULL; - } -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - self = PyTuple_GET_ITEM(args, 0); -#else - self = PySequence_ITEM(args, 0); if (unlikely(!self)) return NULL; -#endif } - if (self && !is_classmethod && !is_staticmethod) { - int is_instance = PyObject_IsInstance(self, binding_func->type); - if (unlikely(!is_instance)) { - PyErr_Format(PyExc_TypeError, - "First argument should be of type %.200s, got %.200s.", - ((PyTypeObject *) binding_func->type)->tp_name, - self->ob_type->tp_name); - goto bad; - } else if (unlikely(is_instance == -1)) { - goto bad; - } - } -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); - self = NULL; -#endif - if (binding_func->__signatures__) { PyObject *tup; if (is_staticmethod && binding_func->func.flags & __Pyx_CYFUNCTION_CCLASS) { @@ -1093,23 +1448,31 @@ if (unlikely(!new_func)) goto bad; - Py_XINCREF(binding_func->func.func_classobj); - Py_CLEAR(new_func->func.func_classobj); - new_func->func.func_classobj = binding_func->func.func_classobj; + __Pyx_CyFunction_SetClassObj(new_func, __Pyx_CyFunction_GetClassObj(binding_func)); func = (PyObject *) new_func; } result = __pyx_FusedFunction_callfunction(func, args, kw); bad: -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); -#endif Py_XDECREF(new_args); Py_XDECREF((PyObject *) new_func); return result; } +static PyObject * +__Pyx_FusedFunction_get_self(__pyx_FusedFunctionObject *m, void *closure) +{ + PyObject *self = m->self; + CYTHON_UNUSED_VAR(closure); + if (unlikely(!self)) { + PyErr_SetString(PyExc_AttributeError, "'function' object has no attribute '__self__'"); + } else { + Py_INCREF(self); + } + return self; +} + static PyMemberDef __pyx_FusedFunction_members[] = { {(char *) "__signatures__", T_OBJECT, @@ -1119,6 +1482,38 @@ {0, 0, 0, 0, 0}, }; +static PyGetSetDef __pyx_FusedFunction_getsets[] = { + {(char *) "__self__", (getter)__Pyx_FusedFunction_get_self, 0, 0, 0}, + // __doc__ is None for the fused function type, but we need it to be + // a descriptor for the instance's __doc__, so rebuild the descriptor in our subclass + // (all other descriptors are inherited) + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {0, 0, 0, 0, 0} +}; + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_FusedFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__pyx_FusedFunction_dealloc}, + {Py_tp_call, (void *)__pyx_FusedFunction_call}, + {Py_tp_traverse, (void *)__pyx_FusedFunction_traverse}, + {Py_tp_clear, (void *)__pyx_FusedFunction_clear}, + {Py_tp_members, (void *)__pyx_FusedFunction_members}, + {Py_tp_getset, (void *)__pyx_FusedFunction_getsets}, + {Py_tp_descr_get, (void *)__pyx_FusedFunction_descr_get}, + {Py_mp_subscript, (void *)__pyx_FusedFunction_getitem}, + {0, 0}, +}; + +static PyType_Spec __pyx_FusedFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", + sizeof(__pyx_FusedFunctionObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_FusedFunctionType_slots +}; + +#else /* !CYTHON_USE_TYPE_SPECS */ + static PyMappingMethods __pyx_FusedFunction_mapping_methods = { 0, (binaryfunc) __pyx_FusedFunction_getitem, @@ -1127,7 +1522,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "fused_cython_function", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", /*tp_name*/ sizeof(__pyx_FusedFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __pyx_FusedFunction_dealloc, /*tp_dealloc*/ @@ -1137,7 +1532,7 @@ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ @@ -1159,9 +1554,8 @@ 0, /*tp_iternext*/ 0, /*tp_methods*/ __pyx_FusedFunction_members, /*tp_members*/ - // __doc__ is None for the fused function type, but we need it to be - // a descriptor for the instance's __doc__, so rebuild descriptors in our subclass - __pyx_CyFunction_getsets, /*tp_getset*/ + __pyx_FusedFunction_getsets, /*tp_getset*/ + // NOTE: tp_base may be changed later during module initialisation when importing CyFunction across modules. &__pyx_CyFunctionType_type, /*tp_base*/ 0, /*tp_dict*/ __pyx_FusedFunction_descr_get, /*tp_descr_get*/ @@ -1182,11 +1576,33 @@ #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + 0, /*tp_vectorcall*/ +#endif +#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 + 0, /*tp_print*/ +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif -static int __pyx_FusedFunction_init(void) { +static int __pyx_FusedFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + PyObject *bases = PyTuple_Pack(1, __pyx_CyFunctionType); + if (unlikely(!bases)) { + return -1; + } + __pyx_FusedFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_FusedFunctionType_spec, bases); + Py_DECREF(bases); +#else + (void) module; + // Set base from __Pyx_FetchCommonTypeFromSpec, in case it's different from the local static value. + __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType; __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type); - if (__pyx_FusedFunctionType == NULL) { +#endif + if (unlikely(__pyx_FusedFunctionType == NULL)) { return -1; } return 0; @@ -1195,31 +1611,36 @@ //////////////////// ClassMethod.proto //////////////////// #include "descrobject.h" -static PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/ +static CYTHON_UNUSED PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/ //////////////////// ClassMethod //////////////////// static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM <= 0x05080000 if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) { // cdef classes return PyClassMethod_New(method); } #else -#if CYTHON_COMPILING_IN_PYSTON - // special C-API function only in Pyston - if (PyMethodDescr_Check(method)) { +#if CYTHON_COMPILING_IN_PYPY + // special C-API function only in PyPy >= 5.9 + if (PyMethodDescr_Check(method)) #else - // It appears that PyMethodDescr_Type is not exposed anywhere in the CPython C-API + #if PY_MAJOR_VERSION == 2 + // PyMethodDescr_Type is not exposed in the CPython C-API in Py2. static PyTypeObject *methoddescr_type = NULL; - if (methoddescr_type == NULL) { + if (unlikely(methoddescr_type == NULL)) { PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append"); - if (!meth) return NULL; + if (unlikely(!meth)) return NULL; methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } - if (PyObject_TypeCheck(method, methoddescr_type)) { + #else + PyTypeObject *methoddescr_type = &PyMethodDescr_Type; + #endif + if (__Pyx_TypeCheck(method, methoddescr_type)) #endif + { // cdef classes PyMethodDescrObject *descr = (PyMethodDescrObject *)method; #if PY_VERSION_HEX < 0x03020000 @@ -1234,16 +1655,7 @@ // python classes return PyClassMethod_New(PyMethod_GET_FUNCTION(method)); } - else if (PyCFunction_Check(method)) { - return PyClassMethod_New(method); - } -#ifdef __Pyx_CyFunction_USED - else if (PyObject_TypeCheck(method, __pyx_CyFunctionType)) { + else { return PyClassMethod_New(method); } -#endif - PyErr_SetString(PyExc_TypeError, - "Class-level classmethod() can only be called on " - "a method_descriptor or instance method."); - return NULL; } diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Dataclasses.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Dataclasses.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Dataclasses.c 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Dataclasses.c 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,78 @@ +///////////////////// ModuleLoader.proto ////////////////////////// + +static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code); /* proto */ + +//////////////////// ModuleLoader /////////////////////// +//@requires: CommonStructures.c::FetchSharedCythonModule + +static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code) { + // We want to be able to use the contents of the standard library dataclasses module where available. + // If those objects aren't available (due to Python version) then a simple fallback is substituted + // instead, which largely just fails with a not-implemented error. + // + // The fallbacks are placed in the "shared abi module" as a convenient internal place to + // store them + + PyObject *shared_abi_module = 0, *module = 0; + + shared_abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!shared_abi_module) return NULL; + + if (PyObject_HasAttrString(shared_abi_module, name)) { + PyObject* result = PyObject_GetAttrString(shared_abi_module, name); + Py_DECREF(shared_abi_module); + return result; + } + + // the best and simplest case is simply to defer to the standard library (if available) + module = PyImport_ImportModule(name); + if (!module) { + PyObject *localDict, *runValue, *builtins, *modulename; + if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad; + PyErr_Clear(); // this is reasonably likely (especially on older versions of Python) +#if PY_MAJOR_VERSION < 3 + modulename = PyBytes_FromFormat("_cython_" CYTHON_ABI ".%s", name); +#else + modulename = PyUnicode_FromFormat("_cython_" CYTHON_ABI ".%s", name); +#endif + if (!modulename) goto bad; +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_CPYTHON + module = PyImport_AddModuleObject(modulename); // borrowed +#else + module = PyImport_AddModule(PyBytes_AsString(modulename)); // borrowed +#endif + Py_DECREF(modulename); + if (!module) goto bad; + Py_INCREF(module); + if (PyObject_SetAttrString(shared_abi_module, name, module) < 0) goto bad; + localDict = PyModule_GetDict(module); // borrowed + if (!localDict) goto bad; + builtins = PyEval_GetBuiltins(); // borrowed + if (!builtins) goto bad; + if (PyDict_SetItemString(localDict, "__builtins__", builtins) <0) goto bad; + + runValue = PyRun_String(fallback_code, Py_file_input, localDict, localDict); + if (!runValue) goto bad; + Py_DECREF(runValue); + } + goto shared_cleanup; + + bad: + Py_CLEAR(module); + shared_cleanup: + Py_XDECREF(shared_abi_module); + return module; +} + +///////////////////// SpecificModuleLoader.proto ////////////////////// +//@substitute: tempita + +static PyObject* __Pyx_Load_{{cname}}_Module(void); /* proto */ + + +//////////////////// SpecificModuleLoader /////////////////////// +//@requires: ModuleLoader + +static PyObject* __Pyx_Load_{{cname}}_Module(void) { + return __Pyx_LoadInternalModule("{{cname}}", {{py_code}}); +} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Dataclasses.py cython-0.20.1+1~202203241016-9537/Cython/Utility/Dataclasses.py --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Dataclasses.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Dataclasses.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,106 @@ +################### Dataclasses_fallback ############################### + +# This is the fallback dataclass code if the stdlib module isn't available. +# It defines enough of the support types to be used with cdef classes +# and to fail if used on regular types. + +# (Intended to be included as py code - not compiled) + +from collections import namedtuple +try: + from types import MappingProxyType +except ImportError: + # mutable fallback if unavailable + MappingProxyType = lambda x: x + +class _MISSING_TYPE(object): + pass +MISSING = _MISSING_TYPE() + +_DataclassParams = namedtuple('_DataclassParams', + ["init", "repr", "eq", "order", "unsafe_hash", "frozen"]) +class Field(object): + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + # Be aware that if MappingProxyType is unavailable (i.e. py2?) then we + # don't enforce non-mutability that the real module does + self.metadata = (MappingProxyType({}) + if metadata is None else + MappingProxyType(metadata)) + self._field_type = None + + def __repr__(self): + return ('Field(' + 'name={0!r},' + 'type={1!r},' + 'default={2!r},' + 'default_factory={3!r},' + 'init={4!r},' + 'repr={5!r},' + 'hash={6!r},' + 'compare={7!r},' + 'metadata={8!r},' + ')'.format(self.name, self.type, self.default, + self.default_factory, self.init, + self.repr, self.hash, self.compare, + self.metadata)) + +# A sentinel object for default values to signal that a default +# factory will be used. This is given a nice repr() which will appear +# in the function signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +def dataclass(*args, **kwds): + raise NotImplementedError("Standard library 'dataclasses' module" + "is unavailable, likely due to the version of Python you're using.") + +# Markers for the various kinds of fields and pseudo-fields. +class _FIELD_BASE: + def __init__(self, name): + self.name = name + def __repr__(self): + return self.name +_FIELD = _FIELD_BASE('_FIELD') +_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') +_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') + +def field(*ignore, **kwds): + default = kwds.pop("default", MISSING) + default_factory = kwds.pop("default_factory", MISSING) + init = kwds.pop("init", True) + repr = kwds.pop("repr", True) + hash = kwds.pop("hash", None) + compare = kwds.pop("compare", True) + metadata = kwds.pop("metadata", None) + + if kwds: + raise ValueError("field received unexpected keyword arguments: %s" + % list(kwds.keys())) + if default is not MISSING and default_factory is not MISSING: + raise ValueError('cannot specify both default and default_factory') + if ignore: + raise ValueError("'field' does not take any positional arguments") + return Field(default, default_factory, init, repr, hash, compare, metadata) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Embed.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Embed.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Embed.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Embed.c 2022-03-24 10:16:46.000000000 +0000 @@ -5,12 +5,13 @@ #endif #if PY_MAJOR_VERSION < 3 -int %(main_method)s(int argc, char** argv) { -#elif defined(WIN32) || defined(MS_WINDOWS) -int %(wmain_method)s(int argc, wchar_t **argv) { +int %(main_method)s(int argc, char** argv) +#elif defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) +int %(wmain_method)s(int argc, wchar_t **argv) #else -static int __Pyx_main(int argc, wchar_t **argv) { +static int __Pyx_main(int argc, wchar_t **argv) #endif +{ /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -24,18 +25,23 @@ #endif if (argc && argv) Py_SetProgramName(argv[0]); + + #if PY_MAJOR_VERSION < 3 + if (PyImport_AppendInittab("%(module_name)s", init%(module_name)s) < 0) return 1; + #else + if (PyImport_AppendInittab("%(module_name)s", PyInit_%(module_name)s) < 0) return 1; + #endif + Py_Initialize(); if (argc && argv) PySys_SetArgv(argc, argv); + { /* init module '%(module_name)s' as '__main__' */ PyObject* m = NULL; %(module_is_main)s = 1; - #if PY_MAJOR_VERSION < 3 - init%(module_name)s(); - #else - m = PyInit_%(module_name)s(); - #endif - if (PyErr_Occurred()) { + m = PyImport_ImportModule("%(module_name)s"); + + if (!m && PyErr_Occurred()) { PyErr_Print(); /* This exits with the right code if SystemExit. */ #if PY_MAJOR_VERSION < 3 if (Py_FlushLine()) PyErr_Clear(); @@ -44,14 +50,21 @@ } Py_XDECREF(m); } +#if PY_VERSION_HEX < 0x03060000 Py_Finalize(); +#else + if (Py_FinalizeEx() < 0) + return 2; +#endif return 0; } -#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS) +#if PY_MAJOR_VERSION >= 3 && !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #include +#if PY_VERSION_HEX < 0x03050000 + static wchar_t* __Pyx_char2wchar(char* arg) { @@ -156,6 +169,8 @@ return NULL; } +#endif + int %(main_method)s(int argc, char **argv) { @@ -178,7 +193,12 @@ res = 0; setlocale(LC_ALL, ""); for (i = 0; i < argc; i++) { - argv_copy2[i] = argv_copy[i] = __Pyx_char2wchar(argv[i]); + argv_copy2[i] = argv_copy[i] = +#if PY_VERSION_HEX < 0x03050000 + __Pyx_char2wchar(argv[i]); +#else + Py_DecodeLocale(argv[i], NULL); +#endif if (!argv_copy[i]) res = 1; /* failure, but continue to simplify cleanup */ } setlocale(LC_ALL, oldloc); @@ -186,7 +206,11 @@ if (res == 0) res = __Pyx_main(argc, argv_copy); for (i = 0; i < argc; i++) { +#if PY_VERSION_HEX < 0x03050000 free(argv_copy2[i]); +#else + PyMem_RawFree(argv_copy2[i]); +#endif } free(argv_copy); free(argv_copy2); diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Exceptions.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Exceptions.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Exceptions.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Exceptions.c 2022-03-24 10:16:46.000000000 +0000 @@ -6,15 +6,34 @@ // __Pyx_GetException() +/////////////// ErrOccurredWithGIL.proto /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ + +/////////////// ErrOccurredWithGIL /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + #ifdef WITH_THREAD + PyGILState_STATE _save = PyGILState_Ensure(); + #endif + err = !!PyErr_Occurred(); + #ifdef WITH_THREAD + PyGILState_Release(_save); + #endif + return err; +} + + /////////////// PyThreadStateGet.proto /////////////// //@substitute: naming #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *$local_tstate_cname; -#define __Pyx_PyThreadState_assign $local_tstate_cname = PyThreadState_GET(); +#define __Pyx_PyThreadState_assign $local_tstate_cname = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() $local_tstate_cname->curexc_type #else #define __Pyx_PyThreadState_declare #define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() #endif @@ -31,11 +50,28 @@ /////////////// PyErrExceptionMatches /////////////// #if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + // the tighter subtype checking in Py3 allows faster out-of-order comparison + for (i=0; icurexc_type; if (exc_type == err) return 1; if (unlikely(!exc_type)) return 0; - return PyErr_GivenExceptionMatches(exc_type, err); + if (unlikely(PyTuple_Check(err))) + return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); } #endif @@ -44,6 +80,7 @@ //@requires: PyThreadStateGet #if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) #define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState($local_tstate_cname, type, value, tb) @@ -51,9 +88,19 @@ static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); /*proto*/ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); /*proto*/ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif + #else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) #define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) #define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) #define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) #endif @@ -97,9 +144,9 @@ // has changed quite a lot between the two versions. #if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { __Pyx_PyThreadState_declare + CYTHON_UNUSED_VAR(cause); /* 'cause' is only used in Py3 */ Py_XINCREF(type); if (!value || value == Py_None) @@ -230,11 +277,7 @@ goto bad; } -#if PY_VERSION_HEX >= 0x03030000 if (cause) { -#else - if (cause && cause != Py_None) { -#endif PyObject *fixed_cause; if (cause == Py_None) { // raise ... from None @@ -258,14 +301,14 @@ PyErr_SetObject(type, value); if (tb) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API PyObject *tmp_type, *tmp_value, *tmp_tb; PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); Py_INCREF(tb); PyErr_Restore(tmp_type, tmp_value, tb); Py_XDECREF(tmp_tb); #else - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = __Pyx_PyThreadState_Current; PyObject* tmp_tb = tstate->curexc_traceback; if (tb != tmp_tb) { Py_INCREF(tb); @@ -281,6 +324,31 @@ } #endif + +/////////////// GetTopmostException.proto /////////////// + +#if CYTHON_USE_EXC_INFO_STACK +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/////////////// GetTopmostException /////////////// + +#if CYTHON_USE_EXC_INFO_STACK +// Copied from errors.c in CPython. +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + + /////////////// GetException.proto /////////////// //@substitute: naming //@requires: PyThreadStateGet @@ -295,10 +363,11 @@ /////////////// GetException /////////////// #if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) #else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) #endif +{ PyObject *local_type, *local_value, *local_tb; #if CYTHON_FAST_THREAD_STATE PyObject *tmp_type, *tmp_value, *tmp_tb; @@ -333,12 +402,33 @@ *value = local_value; *tb = local_tb; #if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = local_type; tstate->exc_value = local_value; tstate->exc_traceback = local_tb; + #endif // Make sure tstate is in a consistent state when we XDECREF // these objects (DECREF may run arbitrary code). Py_XDECREF(tmp_type); @@ -362,34 +452,51 @@ static CYTHON_INLINE void __Pyx_ReraiseException(void); /*proto*/ -/////////////// ReRaiseException.proto /////////////// +/////////////// ReRaiseException /////////////// +//@requires: GetTopmostException static CYTHON_INLINE void __Pyx_ReraiseException(void) { PyObject *type = NULL, *value = NULL, *tb = NULL; #if CYTHON_FAST_THREAD_STATE PyThreadState *tstate = PyThreadState_GET(); + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + value = exc_info->exc_value; + #if PY_VERSION_HEX >= 0x030B00a4 + if (unlikely(value == Py_None)) { + value = NULL; + } else if (value) { + Py_INCREF(value); + type = (PyObject*) Py_TYPE(value); + Py_INCREF(type); + tb = PyException_GetTraceback(value); + } + #else + type = exc_info->exc_type; + tb = exc_info->exc_traceback; + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + #endif + #else type = tstate->exc_type; value = tstate->exc_value; tb = tstate->exc_traceback; + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + #endif #else PyErr_GetExcInfo(&type, &value, &tb); #endif - if (!type || type == Py_None) { -#if !CYTHON_FAST_THREAD_STATE + if (unlikely(!type || type == Py_None)) { Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(tb); -#endif // message copied from Py3 PyErr_SetString(PyExc_RuntimeError, "No active exception to reraise"); } else { -#if CYTHON_FAST_THREAD_STATE - Py_INCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - -#endif PyErr_Restore(type, value, tb); } } @@ -411,28 +518,73 @@ #endif /////////////// SaveResetException /////////////// +//@requires: GetTopmostException #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else *type = tstate->exc_type; *value = tstate->exc_value; *tb = tstate->exc_traceback; Py_XINCREF(*type); Py_XINCREF(*value); Py_XINCREF(*tb); + #endif } static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + // TODO: avoid passing these at all + Py_XDECREF(type); + Py_XDECREF(tb); + #else PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = type; tstate->exc_value = value; tstate->exc_traceback = tb; + #endif Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); + #endif } #endif @@ -452,6 +604,31 @@ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + // TODO: avoid swapping these at all + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + tmp_tb = PyException_GetTraceback(tmp_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; @@ -459,6 +636,7 @@ tstate->exc_type = *type; tstate->exc_value = *value; tstate->exc_traceback = *tb; + #endif *type = tmp_type; *value = tmp_value; @@ -487,9 +665,9 @@ //@requires: PyErrFetchRestore //@requires: PyThreadStateGet -static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { PyObject *old_exc, *old_val, *old_tb; PyObject *ctx; __Pyx_PyThreadState_declare @@ -502,6 +680,11 @@ else state = (PyGILState_STATE)-1; #endif #endif + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); if (full_traceback) { @@ -529,6 +712,66 @@ #endif } +/////////////// CLineInTraceback.proto /////////////// + +#ifdef CYTHON_CLINE_IN_TRACEBACK /* 0 or 1 to disable/enable C line display in tracebacks at C compile time */ +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#else +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);/*proto*/ +#endif + +/////////////// CLineInTraceback /////////////// +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError +//@requires: ObjectHandling.c::PyDictVersioning +//@requires: PyErrFetchRestore +//@substitute: naming + +#ifndef CYTHON_CLINE_IN_TRACEBACK +static int __Pyx_CLineForTraceback(CYTHON_NCP_UNUSED PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + + if (unlikely(!${cython_runtime_cname})) { + // Very early error where the runtime module is not set up yet. + return c_line; + } + + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(${cython_runtime_cname}); + if (likely(cython_runtime_dict)) { + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, PYIDENT("cline_in_traceback"))) + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + // No need to handle errors here when we reset the exception state just afterwards. + (void) PyObject_SetAttr(${cython_runtime_cname}, PYIDENT("cline_in_traceback"), Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + /////////////// AddTraceback.proto /////////////// static void __Pyx_AddTraceback(const char *funcname, int c_line, @@ -536,42 +779,63 @@ /////////////// AddTraceback /////////////// //@requires: ModuleSetupCode.c::CodeObjectCache +//@requires: CLineInTraceback //@substitute: naming #include "compile.h" #include "frameobject.h" #include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + if (c_line) { + // Avoid "unused" warning as long as we don't use this. + (void) $cfilenm_cname; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + _PyTraceback_Add(funcname, filename, py_line); +} +#else static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { - PyCodeObject *py_code = 0; - PyObject *py_srcfile = 0; - PyObject *py_funcname = 0; - + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; #if PY_MAJOR_VERSION < 3 + PyObject *py_srcfile = NULL; + py_srcfile = PyString_FromString(filename); - #else - py_srcfile = PyUnicode_FromString(filename); - #endif if (!py_srcfile) goto bad; + #endif + if (c_line) { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, $cfilenm_cname, c_line); + if (!py_funcname) goto bad; #else py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, $cfilenm_cname, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; #endif } else { #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromString(funcname); - #else - py_funcname = PyUnicode_FromString(funcname); + if (!py_funcname) goto bad; #endif } - if (!py_funcname) goto bad; + #if PY_MAJOR_VERSION < 3 py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ + 0, /*int posonlyargcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ @@ -588,11 +852,16 @@ $empty_bytes /*PyObject *lnotab*/ ); Py_DECREF(py_srcfile); - Py_DECREF(py_funcname); + #else + py_code = PyCode_NewEmpty(filename, funcname, py_line); + #endif + Py_XDECREF(py_funcname); // XDECREF since it's only set on Py3 if cline return py_code; bad: - Py_XDECREF(py_srcfile); Py_XDECREF(py_funcname); + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(py_srcfile); + #endif return NULL; } @@ -600,19 +869,25 @@ int py_line, const char *filename) { PyCodeObject *py_code = 0; PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } - py_code = $global_code_object_cache_find(c_line ? c_line : py_line); + // Negate to avoid collisions between py and c lines. + py_code = $global_code_object_cache_find(c_line ? -c_line : py_line); if (!py_code) { py_code = __Pyx_CreateCodeObjectForTraceback( funcname, c_line, py_line, filename); if (!py_code) goto bad; - $global_code_object_cache_insert(c_line ? c_line : py_line, py_code); + $global_code_object_cache_insert(c_line ? -c_line : py_line, py_code); } py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - $moddict_cname, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + $moddict_cname, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; __Pyx_PyFrame_SetLineNumber(py_frame, py_line); @@ -621,3 +896,4 @@ Py_XDECREF(py_code); Py_XDECREF(py_frame); } +#endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/ExtensionTypes.c cython-0.20.1+1~202203241016-9537/Cython/Utility/ExtensionTypes.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/ExtensionTypes.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/ExtensionTypes.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,3 +1,322 @@ +/////////////// FixUpExtensionType.proto /////////////// + +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); /*proto*/ +#endif + +/////////////// FixUpExtensionType /////////////// +//@requires:ModuleSetupCode.c::IncludeStructmemberH +//@requires:StringTools.c::IncludeStringH + +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if PY_VERSION_HEX > 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + (void) spec; + (void) type; +#else + // Set tp_weakreflist, tp_dictoffset, tp_vectorcalloffset + // Copied and adapted from https://bugs.python.org/issue38140 + const PyType_Slot *slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { + int changed = 0; +#if !(PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON) + const +#endif + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { +#if PY_VERSION_HEX < 0x030900b1 + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#endif +#else + if ((0)); +#endif +#if PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON + else if (strcmp(memb->name, "__module__") == 0) { + // PyType_FromSpec() in CPython <= 3.9b1 overwrites this field with a constant string. + // See https://bugs.python.org/issue40703 + PyObject *descr; + // The PyMemberDef must be an object and normally readable, possibly writable. + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + if (unlikely(PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr) < 0)) { + Py_DECREF(descr); + return -1; + } + Py_DECREF(descr); + changed = 1; + } +#endif + } + memb++; + } + if (changed) + PyType_Modified(type); + } +#endif + return 0; +} +#endif + + +/////////////// ValidateBasesTuple.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); /*proto*/ +#endif + +/////////////// ValidateBasesTuple /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { + // Loop over all bases (except the first) and check that those + // really are heap types. Otherwise, it would not be safe to + // subclass them. + // + // We also check tp_dictoffset: it is unsafe to inherit + // tp_dictoffset from a base class because the object structures + // would not be compatible. So, if our extension type doesn't set + // tp_dictoffset (i.e. there is no __dict__ attribute in the object + // structure), we need to check that none of the base classes sets + // it either. + Py_ssize_t i, n = PyTuple_GET_SIZE(bases); + for (i = 1; i < n; i++) /* Skip first base */ + { + PyObject *b0 = PyTuple_GET_ITEM(bases, i); + PyTypeObject *b; +#if PY_MAJOR_VERSION < 3 + /* Disallow old-style classes */ + if (PyClass_Check(b0)) + { + PyErr_Format(PyExc_TypeError, "base class '%.200s' is an old-style class", + PyString_AS_STRING(((PyClassObject*)b0)->cl_name)); + return -1; + } +#endif + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; + } + if (dictoffset == 0 && b->tp_dictoffset) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; + } + } + return 0; +} +#endif + + +/////////////// PyType_Ready.proto /////////////// + +// unused when using type specs +static CYTHON_UNUSED int __Pyx_PyType_Ready(PyTypeObject *t);/*proto*/ + +/////////////// PyType_Ready /////////////// +//@requires: ObjectHandling.c::PyObjectCallMethod0 +//@requires: ValidateBasesTuple + +// Wrapper around PyType_Ready() with some runtime checks and fixes +// to deal with multiple inheritance. +static int __Pyx_PyType_Ready(PyTypeObject *t) { + +// FIXME: is this really suitable for CYTHON_COMPILING_IN_LIMITED_API? +#if CYTHON_USE_TYPE_SPECS || !(CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API) || defined(PYSTON_MAJOR_VERSION) + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + + return PyType_Ready(t); + +#else + int r; + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; + +#if PY_VERSION_HEX >= 0x03050000 && !defined(PYSTON_MAJOR_VERSION) + { + // Make sure GC does not pick up our non-heap type as heap type with this hack! + // For details, see https://github.com/cython/cython/issues/3603 + int gc_was_enabled; + #if PY_VERSION_HEX >= 0x030A00b1 + // finally added in Py3.10 :) + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + + #else + // Call gc.disable() as a backwards compatible fallback, but only if needed. + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if PY_VERSION_HEX >= 0x030700a1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) + // https://foss.heptapod.net/pypy/pypy/-/issues/3385 + gc = PyImport_GetModule(PYUNICODE("gc")); + #endif + if (unlikely(!gc)) gc = PyImport_Import(PYUNICODE("gc")); + if (unlikely(!gc)) return -1; + py_status = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("isenabled")); + if (unlikely(!py_status)) { + Py_DECREF(gc); + return -1; + } + gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); + Py_DECREF(py_status); + if (gc_was_enabled > 0) { + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("disable")); + if (unlikely(!ret)) { + Py_DECREF(gc); + return -1; + } + Py_DECREF(ret); + } else if (unlikely(gc_was_enabled == -1)) { + Py_DECREF(gc); + return -1; + } + #endif + + // As of https://bugs.python.org/issue22079 + // PyType_Ready enforces that all bases of a non-heap type are + // non-heap. We know that this is the case for the solid base but + // other bases are heap allocated and are kept alive through the + // tp_bases reference. + // Other than this check, the Py_TPFLAGS_HEAPTYPE flag is unused + // in PyType_Ready(). + t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#else + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; +#endif + + r = PyType_Ready(t); + +#if PY_VERSION_HEX >= 0x03050000 && !defined(PYSTON_MAJOR_VERSION) + t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else + if (gc_was_enabled) { + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("enable")); + if (likely(ret || r == -1)) { + Py_XDECREF(ret); + // do not overwrite exceptions raised by PyType_Ready() above + PyErr_Restore(tp, v, tb); + } else { + // PyType_Ready() succeeded, but gc.enable() failed. + Py_XDECREF(tp); + Py_XDECREF(v); + Py_XDECREF(tb); + r = -1; + } + } + Py_DECREF(gc); + #endif + } +#endif + + return r; +#endif +} + + +/////////////// PyTrashcan.proto /////////////// + +// These macros are taken from https://github.com/python/cpython/pull/11841 +// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they +// allow dealing correctly with subclasses. + +// This requires CPython version >= 2.7.4 +// (or >= 3.2.4 but we don't support such old Python 3 versions anyway) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03080000 +// https://github.com/python/cpython/pull/11841 merged so Cython reimplementation +// is no longer necessary +#define __Pyx_TRASHCAN_BEGIN Py_TRASHCAN_BEGIN +#define __Pyx_TRASHCAN_END Py_TRASHCAN_END +#elif CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400 +#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \ + do { \ + PyThreadState *_tstate = NULL; \ + // If "cond" is false, then _tstate remains NULL and the deallocator + // is run normally without involving the trashcan + if (cond) { \ + _tstate = PyThreadState_GET(); \ + if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \ + // Store the object (to be deallocated later) and jump past + // Py_TRASHCAN_END, skipping the body of the deallocator + _PyTrash_thread_deposit_object((PyObject*)(op)); \ + break; \ + } \ + ++_tstate->trash_delete_nesting; \ + } + // The body of the deallocator is here. +#define __Pyx_TRASHCAN_END \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ + } while (0); + +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \ + __Pyx_PyObject_GetSlot(op, tp_dealloc, destructor) == (destructor)(dealloc)) + +#else +// The trashcan is a no-op on other Python implementations +// or old CPython versions +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) +#define __Pyx_TRASHCAN_END +#endif /////////////// CallNextTpDealloc.proto /////////////// @@ -7,13 +326,14 @@ static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc) { PyTypeObject* type = Py_TYPE(obj); + destructor tp_dealloc = NULL; /* try to find the first parent type that has a different tp_dealloc() function */ - while (type && type->tp_dealloc != current_tp_dealloc) - type = type->tp_base; - while (type && type->tp_dealloc == current_tp_dealloc) - type = type->tp_base; + while (type && __Pyx_PyType_GetSlot(type, tp_dealloc, destructor) != current_tp_dealloc) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_dealloc = __Pyx_PyType_GetSlot(type, tp_dealloc, destructor)) == current_tp_dealloc) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); if (type) - type->tp_dealloc(obj); + tp_dealloc(obj); } /////////////// CallNextTpTraverse.proto /////////////// @@ -24,30 +344,202 @@ static int __Pyx_call_next_tp_traverse(PyObject* obj, visitproc v, void *a, traverseproc current_tp_traverse) { PyTypeObject* type = Py_TYPE(obj); + traverseproc tp_traverse = NULL; /* try to find the first parent type that has a different tp_traverse() function */ - while (type && type->tp_traverse != current_tp_traverse) - type = type->tp_base; - while (type && type->tp_traverse == current_tp_traverse) - type = type->tp_base; - if (type && type->tp_traverse) - return type->tp_traverse(obj, v, a); + while (type && __Pyx_PyType_GetSlot(type, tp_traverse, traverseproc) != current_tp_traverse) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_traverse = __Pyx_PyType_GetSlot(type, tp_traverse, traverseproc)) == current_tp_traverse) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + if (type && tp_traverse) + return tp_traverse(obj, v, a); // FIXME: really ignore? return 0; } /////////////// CallNextTpClear.proto /////////////// -static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_dealloc); +static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear); /////////////// CallNextTpClear /////////////// static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) { PyTypeObject* type = Py_TYPE(obj); + inquiry tp_clear = NULL; /* try to find the first parent type that has a different tp_clear() function */ - while (type && type->tp_clear != current_tp_clear) - type = type->tp_base; - while (type && type->tp_clear == current_tp_clear) - type = type->tp_base; - if (type && type->tp_clear) - type->tp_clear(obj); + while (type && __Pyx_PyType_GetSlot(type, tp_clear, inquiry) != current_tp_clear) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_clear = __Pyx_PyType_GetSlot(type, tp_clear, inquiry)) == current_tp_clear) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + if (type && tp_clear) + tp_clear(obj); +} + +/////////////// SetupReduce.proto /////////////// + +#if !CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_setup_reduce(PyObject* type_obj); +#endif + +/////////////// SetupReduce /////////////// +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError +//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@substitute: naming + +#if !CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, PYIDENT("__name__")); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + + Py_XDECREF(name_attr); + return ret; +} + +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; + +#if CYTHON_USE_PYTYPE_LOOKUP + if (_PyType_Lookup((PyTypeObject*)type_obj, PYIDENT("__getstate__"))) goto __PYX_GOOD; +#else + if (PyObject_HasAttr(type_obj, PYIDENT("__getstate__"))) goto __PYX_GOOD; +#endif + +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, PYIDENT("__reduce_ex__")); if (!object_reduce_ex) goto __PYX_BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, PYIDENT("__reduce_ex__")); if (!object_reduce_ex) goto __PYX_BAD; +#endif + + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, PYIDENT("__reduce_ex__")); if (unlikely(!reduce_ex)) goto __PYX_BAD; + if (reduce_ex == object_reduce_ex) { + +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, PYIDENT("__reduce__")); if (!object_reduce) goto __PYX_BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, PYIDENT("__reduce__")); if (!object_reduce) goto __PYX_BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, PYIDENT("__reduce__")); if (unlikely(!reduce)) goto __PYX_BAD; + + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, PYIDENT("__reduce_cython__"))) { + reduce_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__reduce_cython__")); + if (likely(reduce_cython)) { + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, PYIDENT("__reduce__"), reduce_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, PYIDENT("__reduce_cython__")); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (reduce == object_reduce || PyErr_Occurred()) { + // Ignore if we're done, i.e. if 'reduce' already has the right name and the original is gone. + // Otherwise: error. + goto __PYX_BAD; + } + + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate__")); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, PYIDENT("__setstate_cython__"))) { + setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate_cython__")); + if (likely(setstate_cython)) { + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, PYIDENT("__setstate__"), setstate_cython); if (unlikely(ret < 0)) goto __PYX_BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, PYIDENT("__setstate_cython__")); if (unlikely(ret < 0)) goto __PYX_BAD; + } else if (!setstate || PyErr_Occurred()) { + // Ignore if we're done, i.e. if 'setstate' already has the right name and the original is gone. + // Otherwise: error. + goto __PYX_BAD; + } + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto __PYX_GOOD; + +__PYX_BAD: + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } + ret = -1; +__PYX_GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} +#endif + + +/////////////// BinopSlot /////////////// + +static CYTHON_INLINE PyObject *{{func_name}}_maybe_call_slot(PyTypeObject* type, PyObject *left, PyObject *right {{extra_arg_decl}}) { + {{slot_type}} slot; +#if CYTHON_USE_TYPE_SLOTS || PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY + slot = type->tp_as_number ? type->tp_as_number->{{slot_name}} : NULL; +#else + slot = ({{slot_type}}) PyType_GetSlot(type, Py_{{slot_name}}); +#endif + return slot ? slot(left, right {{extra_arg}}) : __Pyx_NewRef(Py_NotImplemented); +} + +static PyObject *{{func_name}}(PyObject *left, PyObject *right {{extra_arg_decl}}) { + int maybe_self_is_left, maybe_self_is_right = 0; + maybe_self_is_left = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(left)->tp_as_number && Py_TYPE(left)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(left, {{type_cname}}); + // Optimize for the common case where the left operation is defined (and successful). + if (!({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(right, {{type_cname}}); + } + if (maybe_self_is_left) { + PyObject *res; + if (maybe_self_is_right && {{overloads_right}} && !({{overloads_left}})) { + res = {{call_right}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + // Don't bother calling it again. + maybe_self_is_right = 0; + } + res = {{call_left}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + } + if (({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || PyType_IsSubtype(Py_TYPE(right), {{type_cname}}); + } + if (maybe_self_is_right) { + return {{call_right}}; + } + return __Pyx_NewRef(Py_NotImplemented); } diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/FunctionArguments.c cython-0.20.1+1~202203241016-9537/Cython/Utility/FunctionArguments.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/FunctionArguments.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/FunctionArguments.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,34 +1,37 @@ //////////////////// ArgTypeTest.proto //////////////////// -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); /*proto*/ -//////////////////// ArgTypeTest //////////////////// +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact) \ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 : \ + __Pyx__ArgTypeTest(obj, type, name, exact)) -static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); /*proto*/ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) +//////////////////// ArgTypeTest //////////////////// + +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } - if (none_allowed && obj == Py_None) return 1; else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; #endif } else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; + if (likely(__Pyx_TypeCheck(obj, type))) return 1; } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + type_name = __Pyx_PyType_GetName(type); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")", name, type_name, obj_type_name); + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); return 0; } @@ -72,14 +75,11 @@ //////////////////// RaiseKeywordRequired.proto //////////////////// -static CYTHON_INLINE void __Pyx_RaiseKeywordRequired(const char* func_name, PyObject* kw_name); /*proto*/ +static void __Pyx_RaiseKeywordRequired(const char* func_name, PyObject* kw_name); /*proto*/ //////////////////// RaiseKeywordRequired //////////////////// -static CYTHON_INLINE void __Pyx_RaiseKeywordRequired( - const char* func_name, - PyObject* kw_name) -{ +static void __Pyx_RaiseKeywordRequired(const char* func_name, PyObject* kw_name) { PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION >= 3 "%s() needs keyword-only argument %U", func_name, kw_name); @@ -117,22 +117,28 @@ //////////////////// RaiseMappingExpected //////////////////// static void __Pyx_RaiseMappingExpectedError(PyObject* arg) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(arg)->tp_name); + __Pyx_TypeName arg_type_name = __Pyx_PyType_GetName(Py_TYPE(arg)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not a mapping", arg_type_name); + __Pyx_DECREF_TypeName(arg_type_name); } //////////////////// KeywordStringCheck.proto //////////////////// -static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/ +static int __Pyx_CheckKeywordStrings(PyObject *kw, const char* function_name, int kw_allowed); /*proto*/ //////////////////// KeywordStringCheck //////////////////// -// __Pyx_CheckKeywordStrings raises an error if non-string keywords -// were passed to a function, or if any keywords were passed to a -// function that does not accept them. +// __Pyx_CheckKeywordStrings raises an error if non-string keywords +// were passed to a function, or if any keywords were passed to a +// function that does not accept them. +// +// The "kw" argument is either a dict (for METH_VARARGS) or a tuple +// (for METH_FASTCALL). -static CYTHON_INLINE int __Pyx_CheckKeywordStrings( - PyObject *kwdict, +static int __Pyx_CheckKeywordStrings( + PyObject *kw, const char* function_name, int kw_allowed) { @@ -140,18 +146,37 @@ Py_ssize_t pos = 0; #if CYTHON_COMPILING_IN_PYPY /* PyPy appears to check keywords at call time, not at unpacking time => not much to do here */ - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + if (!kw_allowed && PyDict_Next(kw, &pos, &key, 0)) goto invalid_keyword; return 1; #else - while (PyDict_Next(kwdict, &pos, &key, 0)) { + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kw))) { + if (unlikely(PyTuple_GET_SIZE(kw) == 0)) + return 1; + if (!kw_allowed) { + key = PyTuple_GET_ITEM(kw, 0); + goto invalid_keyword; + } +#if PY_VERSION_HEX < 0x03090000 + // On CPython >= 3.9, the FASTCALL protocol guarantees that keyword + // names are strings (see https://bugs.python.org/issue37540) + for (pos = 0; pos < PyTuple_GET_SIZE(kw); pos++) { + key = PyTuple_GET_ITEM(kw, pos); + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; + } +#endif + return 1; + } + + while (PyDict_Next(kw, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 - if (unlikely(!PyString_CheckExact(key)) && unlikely(!PyString_Check(key))) + if (unlikely(!PyString_Check(key))) #endif if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; } - if ((!kw_allowed) && unlikely(key)) + if (!kw_allowed && unlikely(key)) goto invalid_keyword; return 1; invalid_keyword_type: @@ -160,11 +185,12 @@ return 0; #endif invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -174,17 +200,22 @@ //////////////////// ParseKeywords.proto //////////////////// -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject *const *kwvalues, + PyObject **argnames[], + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ //////////////////// ParseKeywords //////////////////// //@requires: RaiseDoubleKeywords // __Pyx_ParseOptionalKeywords copies the optional/unknown keyword -// arguments from the kwds dict into kwds2. If kwds2 is NULL, unknown +// arguments from kwds into the dict kwds2. If kwds2 is NULL, unknown // keywords will raise an invalid keyword error. // +// When not using METH_FASTCALL, kwds is a dict and kwvalues is NULL. +// Otherwise, kwds is a tuple with keyword names and kwvalues is a C +// array with the corresponding values. +// // Three kinds of errors are checked: 1) non-string keywords, 2) // unexpected keywords and 3) overlap with positional arguments. // @@ -196,6 +227,7 @@ static int __Pyx_ParseOptionalKeywords( PyObject *kwds, + PyObject *const *kwvalues, PyObject **argnames[], PyObject *kwds2, PyObject *values[], @@ -206,8 +238,20 @@ Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; + int kwds_is_tuple = CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)); + + while (1) { + if (kwds_is_tuple) { + if (pos >= PyTuple_GET_SIZE(kwds)) break; + key = PyTuple_GET_ITEM(kwds, pos); + value = kwvalues[pos]; + pos++; + } + else + { + if (!PyDict_Next(kwds, &pos, &key, &value)) break; + } - while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { @@ -217,7 +261,7 @@ name = first_kw_arg; #if PY_MAJOR_VERSION < 3 - if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + if (likely(PyString_Check(key))) { while (*name) { if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) && _PyString_Eq(**name, key)) { @@ -243,12 +287,13 @@ #endif if (likely(PyUnicode_Check(key))) { while (*name) { - int cmp = (**name == key) ? 0 : + int cmp = ( #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + (__Pyx_PyUnicode_GET_LENGTH(**name) != __Pyx_PyUnicode_GET_LENGTH(key)) ? 1 : #endif - // need to convert argument name from bytes to unicode for comparison - PyUnicode_Compare(**name, key); + // In Py2, we may need to convert the argument name from str to unicode for comparison. + PyUnicode_Compare(**name, key) + ); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; @@ -263,7 +308,7 @@ while (argname != first_kw_arg) { int cmp = (**argname == key) ? 0 : #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 - (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + (__Pyx_PyUnicode_GET_LENGTH(**argname) != __Pyx_PyUnicode_GET_LENGTH(key)) ? 1 : #endif // need to convert argument name from bytes to unicode for comparison PyUnicode_Compare(**argname, key); @@ -290,11 +335,12 @@ "%.200s() keywords must be strings", function_name); goto bad; invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -320,7 +366,7 @@ if (unlikely(!iter)) { // slow fallback: try converting to dict, then iterate PyObject *args; - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad; PyErr_Clear(); args = PyTuple_Pack(1, source_mapping); if (likely(args)) { @@ -356,3 +402,74 @@ Py_XDECREF(iter); return -1; } + + +/////////////// fastcall.proto /////////////// + +// We define various functions and macros with two variants: +//..._FASTCALL and ..._VARARGS + +// The first is used when METH_FASTCALL is enabled and the second is used +// otherwise. If the Python implementation does not support METH_FASTCALL +// (because it's an old version of CPython or it's not CPython at all), +// then the ..._FASTCALL macros simply alias ..._VARARGS + +#define __Pyx_Arg_VARARGS(args, i) PyTuple_GET_ITEM(args, i) +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_Arg_FASTCALL(args, i) args[i] + #define __Pyx_NumKwargs_FASTCALL(kwds) PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) (&args[nargs]) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) +#else + #define __Pyx_Arg_FASTCALL __Pyx_Arg_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_VARARGS(args, start), stop - start) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_FASTCALL(args, start), stop - start) +#else +/* Not CPython, so certainly no METH_FASTCALL support */ +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + + +/////////////// fastcall /////////////// +//@requires: ObjectHandling.c::TupleAndListFromArray +//@requires: StringTools.c::UnicodeEquals + +#if CYTHON_METH_FASTCALL +// kwnames: tuple with names of keyword arguments +// kwvalues: C array with values of keyword arguments +// s: str with the keyword name to look for +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + // Search the kwnames array for s and return the corresponding value. + // We do two loops: a first one to compare pointers (which will find a + // match if the name in kwnames is interned, given that s is interned + // by Cython). A second loop compares the actual strings. + Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < n; i++) + { + if (s == PyTuple_GET_ITEM(kwnames, i)) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + int eq = __Pyx_PyUnicode_Equals(s, PyTuple_GET_ITEM(kwnames, i), Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; // error + return kwvalues[i]; + } + } + return NULL; // not found (no exception set) +} +#endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/ImportExport.c cython-0.20.1+1~202203241016-9537/Cython/Utility/ImportExport.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/ImportExport.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/ImportExport.c 2022-03-24 10:16:46.000000000 +0000 @@ -9,6 +9,147 @@ #endif +/////////////// ImportDottedModule.proto /////////////// + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); /*proto*/ + +/////////////// ImportDottedModule /////////////// +//@requires: Import + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } + if (likely(PyTuple_GET_SIZE(parts_tuple) == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + + PyErr_Format( +#if PY_MAJOR_VERSION < 3 + PyExc_ImportError, + "No module named '%s'", PyString_AS_STRING(partial_name)); +#else +#if PY_VERSION_HEX >= 0x030600B1 + PyExc_ModuleNotFoundError, +#else + PyExc_ImportError, +#endif + "No module named '%U'", partial_name); +#endif + +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +#endif + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +#endif + +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if PY_MAJOR_VERSION < 3 + PyObject *module, *from_list, *star = PYIDENT("*"); + CYTHON_UNUSED_VAR(parts_tuple); + from_list = PyList_New(1); + if (unlikely(!from_list)) + return NULL; + Py_INCREF(star); + PyList_SET_ITEM(from_list, 0, star); + module = __Pyx_Import(name, from_list, 0); + Py_DECREF(from_list); + return module; +#else + Py_ssize_t i, nparts; + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + + // Look up module in sys.modules, which is safer than the attribute lookups below. + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + + nparts = PyTuple_GET_SIZE(parts_tuple); + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = PySequence_ITEM(parts_tuple, i); +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (likely(module)) + return module; + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); +#endif +} + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030400B1 + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + // CPython guards against thread-concurrent initialisation in importlib. + // In this case, we let PyImport_ImportModuleLevelObject() handle the locking. + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, PYIDENT("__spec__")); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, PYIDENT("_initializing")); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + // Not in initialisation phase => use modules as is. + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + + return __Pyx__ImportDottedModule(name, parts_tuple); +} + + /////////////// Import.proto /////////////// static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/ @@ -18,49 +159,39 @@ //@substitute: naming static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; PyObject *module = 0; - PyObject *global_dict = 0; PyObject *empty_dict = 0; - PyObject *list; - #if PY_VERSION_HEX < 0x03030000 + PyObject *empty_list = 0; + #if PY_MAJOR_VERSION < 3 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__")); - if (!py_import) + if (unlikely(!py_import)) goto bad; - #endif - if (from_list) - list = from_list; - else { + if (!from_list) { empty_list = PyList_New(0); - if (!empty_list) + if (unlikely(!empty_list)) goto bad; - list = empty_list; + from_list = empty_list; } - global_dict = PyModule_GetDict($module_cname); - if (!global_dict) - goto bad; + #endif empty_dict = PyDict_New(); - if (!empty_dict) + if (unlikely(!empty_dict)) goto bad; { #if PY_MAJOR_VERSION >= 3 if (level == -1) { - if (strchr(__Pyx_MODULE_NAME, '.')) { + // Avoid C compiler warning if strchr() evaluates to false at compile time. + if ((1) && (strchr(__Pyx_MODULE_NAME, '.'))) { /* try package relative import first */ - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, 1); #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); + name, $moddict_cname, empty_dict, from_list, 1); #endif - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) goto bad; PyErr_Clear(); } @@ -69,25 +200,30 @@ } #endif if (!module) { - #if PY_VERSION_HEX < 0x03030000 + #if PY_MAJOR_VERSION < 3 PyObject *py_level = PyInt_FromLong(level); - if (!py_level) + if (unlikely(!py_level)) goto bad; module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); + name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, level); + #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); + name, $moddict_cname, empty_dict, from_list, level); + #endif #endif } } bad: - #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + #if PY_MAJOR_VERSION < 3 Py_XDECREF(py_import); #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); return module; } @@ -102,6 +238,39 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + // 'name' may refer to a (sub-)module which has not finished initialization + // yet, and may not be assigned as an attribute to its parent, so try + // finding it by full name. + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, PYUNICODE(".")); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { PyErr_Format(PyExc_ImportError, #if PY_MAJOR_VERSION < 3 "cannot import name %.230s", PyString_AS_STRING(name)); @@ -160,11 +329,12 @@ } if (skip_leading_underscores && #if PY_MAJOR_VERSION < 3 - PyString_Check(name) && + likely(PyString_Check(name)) && PyString_AS_STRING(name)[0] == '_') #else - PyUnicode_Check(name) && - PyUnicode_AS_UNICODE(name)[0] == '_') + likely(PyUnicode_Check(name)) && + likely(__Pyx_PyUnicode_GET_LENGTH(name)) && + __Pyx_PyUnicode_READ_CHAR(name, 0) == '_') #endif { Py_DECREF(name); @@ -231,35 +401,10 @@ } -/////////////// ModuleImport.proto /////////////// - -static PyObject *__Pyx_ImportModule(const char *name); /*proto*/ - -/////////////// ModuleImport /////////////// -//@requires: PyIdentifierFromString - -#ifndef __PYX_HAVE_RT_ImportModule -#define __PYX_HAVE_RT_ImportModule -static PyObject *__Pyx_ImportModule(const char *name) { - PyObject *py_name = 0; - PyObject *py_module = 0; - - py_name = __Pyx_PyIdentifier_FromString(name); - if (!py_name) - goto bad; - py_module = PyImport_Import(py_name); - Py_DECREF(py_name); - return py_module; -bad: - Py_XDECREF(py_name); - return 0; -} -#endif - - /////////////// SetPackagePathFromImportLib.proto /////////////// -#if PY_VERSION_HEX >= 0x03030000 +// PY_VERSION_HEX >= 0x03030000 +#if PY_MAJOR_VERSION >= 3 && !CYTHON_PEP489_MULTI_PHASE_INIT static int __Pyx_SetPackagePathFromImportLib(const char* parent_package_name, PyObject *module_name); #else #define __Pyx_SetPackagePathFromImportLib(a, b) 0 @@ -269,7 +414,8 @@ //@requires: ObjectHandling.c::PyObjectGetAttrStr //@substitute: naming -#if PY_VERSION_HEX >= 0x03030000 +// PY_VERSION_HEX >= 0x03030000 +#if PY_MAJOR_VERSION >= 3 && !CYTHON_PEP489_MULTI_PHASE_INIT static int __Pyx_SetPackagePathFromImportLib(const char* parent_package_name, PyObject *module_name) { PyObject *importlib, *loader, *osmod, *ossep, *parts, *package_path; PyObject *path = NULL, *file_path = NULL; @@ -341,37 +487,34 @@ /////////////// TypeImport.proto /////////////// -static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); /*proto*/ +#ifndef __PYX_HAVE_RT_ImportType_proto +#define __PYX_HAVE_RT_ImportType_proto + +enum __Pyx_ImportType_CheckSize { + __Pyx_ImportType_CheckSize_Error = 0, + __Pyx_ImportType_CheckSize_Warn = 1, + __Pyx_ImportType_CheckSize_Ignore = 2 +}; + +static PyTypeObject *__Pyx_ImportType(PyObject* module, const char *module_name, const char *class_name, size_t size, enum __Pyx_ImportType_CheckSize check_size); /*proto*/ + +#endif /////////////// TypeImport /////////////// -//@requires: PyIdentifierFromString -//@requires: ModuleImport #ifndef __PYX_HAVE_RT_ImportType #define __PYX_HAVE_RT_ImportType -static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, - size_t size, int strict) +static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, const char *class_name, + size_t size, enum __Pyx_ImportType_CheckSize check_size) { - PyObject *py_module = 0; PyObject *result = 0; - PyObject *py_name = 0; char warning[200]; Py_ssize_t basicsize; -#ifdef Py_LIMITED_API +#if CYTHON_COMPILING_IN_LIMITED_API PyObject *py_basicsize; #endif - py_module = __Pyx_ImportModule(module_name); - if (!py_module) - goto bad; - py_name = __Pyx_PyIdentifier_FromString(class_name); - if (!py_name) - goto bad; - result = PyObject_GetAttr(py_module, py_name); - Py_DECREF(py_name); - py_name = 0; - Py_DECREF(py_module); - py_module = 0; + result = PyObject_GetAttrString(module, class_name); if (!result) goto bad; if (!PyType_Check(result)) { @@ -380,7 +523,7 @@ module_name, class_name); goto bad; } -#ifndef Py_LIMITED_API +#if !CYTHON_COMPILING_IN_LIMITED_API basicsize = ((PyTypeObject *)result)->tp_basicsize; #else py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); @@ -392,21 +535,30 @@ if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) goto bad; #endif - if (!strict && (size_t)basicsize > size) { - PyOS_snprintf(warning, sizeof(warning), - "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd", - module_name, class_name, basicsize, size); - if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; + if ((size_t)basicsize < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize); + goto bad; } - else if ((size_t)basicsize != size) { + if (check_size == __Pyx_ImportType_CheckSize_Error && (size_t)basicsize != size) { PyErr_Format(PyExc_ValueError, - "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd", - module_name, class_name, basicsize, size); + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize); goto bad; } + else if (check_size == __Pyx_ImportType_CheckSize_Warn && (size_t)basicsize > size) { + PyOS_snprintf(warning, sizeof(warning), + "%s.%s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize); + if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad; + } + /* check_size == __Pyx_ImportType_CheckSize_Ignore does not warn nor error */ return (PyTypeObject *)result; bad: - Py_XDECREF(py_module); Py_XDECREF(result); return NULL; } @@ -439,7 +591,6 @@ PyModule_GetName(module), funcname); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -447,21 +598,6 @@ goto bad; } tmp.p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), funcname, sig, desc); - goto bad; - } - tmp.p = PyCObject_AsVoidPtr(cobj);} -#endif *f = tmp.fp; if (!(*f)) goto bad; @@ -499,11 +635,7 @@ goto bad; } tmp.fp = f; -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(tmp.p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(tmp.p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItemString(d, name, cobj) < 0) @@ -540,7 +672,6 @@ PyModule_GetName(module), name); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -548,21 +679,6 @@ goto bad; } *p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), name, sig, desc); - goto bad; - } - *p = PyCObject_AsVoidPtr(cobj);} -#endif if (!(*p)) goto bad; Py_DECREF(d); @@ -594,11 +710,7 @@ if (__Pyx_PyObject_SetAttrStr($module_cname, PYIDENT("$api_name"), d) < 0) goto bad; } -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItem(d, name, cobj) < 0) @@ -615,19 +727,19 @@ /////////////// SetVTable.proto /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); /*proto*/ /////////////// SetVTable /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, PYIDENT("__pyx_vtable__"), ob) < 0)) #else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); + if (unlikely(PyDict_SetItem(type->tp_dict, PYIDENT("__pyx_vtable__"), ob) < 0)) #endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0) goto bad; Py_DECREF(ob); return 0; @@ -639,20 +751,20 @@ /////////////// GetVTable.proto /////////////// -static void* __Pyx_GetVtable(PyObject *dict); /*proto*/ +static void* __Pyx_GetVtable(PyTypeObject *type); /*proto*/ /////////////// GetVTable /////////////// -static void* __Pyx_GetVtable(PyObject *dict) { +static void* __Pyx_GetVtable(PyTypeObject *type) { void* ptr; - PyObject *ob = PyObject_GetItem(dict, PYIDENT("__pyx_vtable__")); +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, PYIDENT("__pyx_vtable__")); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, PYIDENT("__pyx_vtable__")); +#endif if (!ob) goto bad; -#if PY_VERSION_HEX >= 0x02070000 ptr = PyCapsule_GetPointer(ob, 0); -#else - ptr = PyCObject_AsVoidPtr(ob); -#endif if (!ptr && !PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); Py_DECREF(ob); @@ -661,3 +773,113 @@ Py_XDECREF(ob); return NULL; } + + +/////////////// MergeVTables.proto /////////////// +//@requires: GetVTable + +// TODO: find a way to make this work with the Limited API! +#if !CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_MergeVtables(PyTypeObject *type); /*proto*/ +#endif + +/////////////// MergeVTables /////////////// + +#if !CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_MergeVtables(PyTypeObject *type) { + int i; + void** base_vtables; + __Pyx_TypeName tp_base_name; + __Pyx_TypeName base_name; + void* unknown = (void*)-1; + PyObject* bases = type->tp_bases; + int base_depth = 0; + { + PyTypeObject* base = type->tp_base; + while (base) { + base_depth += 1; + base = base->tp_base; + } + } + base_vtables = (void**) malloc(sizeof(void*) * (size_t)(base_depth + 1)); + base_vtables[0] = unknown; + // Could do MRO resolution of individual methods in the future, assuming + // compatible vtables, but for now simply require a common vtable base. + // Note that if the vtables of various bases are extended separately, + // resolution isn't possible and we must reject it just as when the + // instance struct is so extended. (It would be good to also do this + // check when a multiple-base class is created in pure Python as well.) + for (i = 1; i < PyTuple_GET_SIZE(bases); i++) { + void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))); + if (base_vtable != NULL) { + int j; + PyTypeObject* base = type->tp_base; + for (j = 0; j < base_depth; j++) { + if (base_vtables[j] == unknown) { + base_vtables[j] = __Pyx_GetVtable(base); + base_vtables[j + 1] = unknown; + } + if (base_vtables[j] == base_vtable) { + break; + } else if (base_vtables[j] == NULL) { + // No more potential matching bases (with vtables). + goto bad; + } + base = base->tp_base; + } + } + } + PyErr_Clear(); + free(base_vtables); + return 0; +bad: + tp_base_name = __Pyx_PyType_GetName(type->tp_base); + base_name = __Pyx_PyType_GetName((PyTypeObject*)PyTuple_GET_ITEM(bases, i)); + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); + free(base_vtables); + return -1; +} +#endif + + +/////////////// ImportNumPyArray.proto /////////////// + +static PyObject *__pyx_numpy_ndarray = NULL; + +static PyObject* __Pyx_ImportNumPyArrayTypeIfAvailable(void); /*proto*/ + +/////////////// ImportNumPyArray.cleanup /////////////// +Py_CLEAR(__pyx_numpy_ndarray); + +/////////////// ImportNumPyArray /////////////// +//@requires: ImportExport.c::Import + +static PyObject* __Pyx__ImportNumPyArray(void) { + PyObject *numpy_module, *ndarray_object = NULL; + numpy_module = __Pyx_Import(PYIDENT("numpy"), NULL, 0); + if (likely(numpy_module)) { + ndarray_object = PyObject_GetAttrString(numpy_module, "ndarray"); + Py_DECREF(numpy_module); + } + if (unlikely(!ndarray_object)) { + // ImportError, AttributeError, ... + PyErr_Clear(); + } + if (unlikely(!ndarray_object || !PyObject_TypeCheck(ndarray_object, &PyType_Type))) { + Py_XDECREF(ndarray_object); + Py_INCREF(Py_None); + ndarray_object = Py_None; + } + return ndarray_object; +} + +static CYTHON_INLINE PyObject* __Pyx_ImportNumPyArrayTypeIfAvailable(void) { + if (unlikely(!__pyx_numpy_ndarray)) { + __pyx_numpy_ndarray = __Pyx__ImportNumPyArray(); + } + Py_INCREF(__pyx_numpy_ndarray); + return __pyx_numpy_ndarray; +} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/MemoryView_C.c cython-0.20.1+1~202203241016-9537/Cython/Utility/MemoryView_C.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/MemoryView_C.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/MemoryView_C.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ ////////// MemviewSliceStruct.proto ////////// +//@proto_block: utility_code_proto_before_types /* memoryview slice struct */ struct {{memview_struct_name}}; @@ -11,8 +12,12 @@ Py_ssize_t suboffsets[{{max_dims}}]; } {{memviewslice_name}}; +// used for "len(memviewslice)" +#define __Pyx_MemoryView_Len(m) (m.shape[0]) + /////////// Atomics.proto ///////////// +//@proto_block: utility_code_proto_before_types #include @@ -77,7 +82,7 @@ /////////////// ObjectToMemviewSlice.proto /////////////// -static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *); +static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *, int writable_flag); ////////// MemviewSliceInit.proto ////////// @@ -108,9 +113,9 @@ #define __pyx_get_slice_count_pointer(memview) (memview->acquisition_count_aligned_p) #define __pyx_get_slice_count(memview) (*__pyx_get_slice_count_pointer(memview)) #define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) -#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int); -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *, int, int); /////////////// MemviewSliceIndex.proto /////////////// @@ -122,7 +127,7 @@ /////////////// ObjectToMemviewSlice /////////////// //@requires: MemviewSliceValidateAndInit -static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj) { +static CYTHON_INLINE {{memviewslice_name}} {{funcname}}(PyObject *obj, int writable_flag) { {{memviewslice_name}} result = {{memslice_init}}; __Pyx_BufFmt_StackElem stack[{{struct_nesting_depth}}]; int axes_specs[] = { {{axes_specs}} }; @@ -135,7 +140,7 @@ } retcode = __Pyx_ValidateAndInit_memviewslice(axes_specs, {{c_or_f_flag}}, - {{buf_flag}}, {{ndim}}, + {{buf_flag}} | writable_flag, {{ndim}}, &{{dtype_typeinfo}}, stack, &result, obj); @@ -164,6 +169,8 @@ /////////////// MemviewSliceValidateAndInit /////////////// //@requires: Buffer.c::TypeInfoCompare +//@requires: Buffer.c::BufferFormatStructs +//@requires: Buffer.c::BufferFormatCheck static int __pyx_check_strides(Py_buffer *buf, int dim, int ndim, int spec) @@ -174,13 +181,13 @@ if (buf->strides) { if (spec & __Pyx_MEMVIEW_CONTIG) { if (spec & (__Pyx_MEMVIEW_PTR|__Pyx_MEMVIEW_FULL)) { - if (buf->strides[dim] != sizeof(void *)) { + if (unlikely(buf->strides[dim] != sizeof(void *))) { PyErr_Format(PyExc_ValueError, "Buffer is not indirectly contiguous " "in dimension %d.", dim); goto fail; } - } else if (buf->strides[dim] != buf->itemsize) { + } else if (unlikely(buf->strides[dim] != buf->itemsize)) { PyErr_SetString(PyExc_ValueError, "Buffer and memoryview are not contiguous " "in the same dimension."); @@ -192,7 +199,7 @@ Py_ssize_t stride = buf->strides[dim]; if (stride < 0) stride = -stride; - if (stride < buf->itemsize) { + if (unlikely(stride < buf->itemsize)) { PyErr_SetString(PyExc_ValueError, "Buffer and memoryview are not contiguous " "in the same dimension."); @@ -200,17 +207,17 @@ } } } else { - if (spec & __Pyx_MEMVIEW_CONTIG && dim != ndim - 1) { + if (unlikely(spec & __Pyx_MEMVIEW_CONTIG && dim != ndim - 1)) { PyErr_Format(PyExc_ValueError, "C-contiguous buffer is not contiguous in " "dimension %d", dim); goto fail; - } else if (spec & (__Pyx_MEMVIEW_PTR)) { + } else if (unlikely(spec & (__Pyx_MEMVIEW_PTR))) { PyErr_Format(PyExc_ValueError, "C-contiguous buffer is not indirect in " "dimension %d", dim); goto fail; - } else if (buf->suboffsets) { + } else if (unlikely(buf->suboffsets)) { PyErr_SetString(PyExc_ValueError, "Buffer exposes suboffsets but no strides"); goto fail; @@ -223,12 +230,13 @@ } static int -__pyx_check_suboffsets(Py_buffer *buf, int dim, CYTHON_UNUSED int ndim, int spec) +__pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) { + CYTHON_UNUSED_VAR(ndim); // Todo: without PyBUF_INDIRECT we may not have suboffset information, i.e., the // ptr may not be set to NULL but may be uninitialized? if (spec & __Pyx_MEMVIEW_DIRECT) { - if (buf->suboffsets && buf->suboffsets[dim] >= 0) { + if (unlikely(buf->suboffsets && buf->suboffsets[dim] >= 0)) { PyErr_Format(PyExc_ValueError, "Buffer not compatible with direct access " "in dimension %d.", dim); @@ -237,7 +245,7 @@ } if (spec & __Pyx_MEMVIEW_PTR) { - if (!buf->suboffsets || (buf->suboffsets && buf->suboffsets[dim] < 0)) { + if (unlikely(!buf->suboffsets || (buf->suboffsets[dim] < 0))) { PyErr_Format(PyExc_ValueError, "Buffer is not indirectly accessible " "in dimension %d.", dim); @@ -258,9 +266,7 @@ if (c_or_f_flag & __Pyx_IS_F_CONTIG) { Py_ssize_t stride = 1; for (i = 0; i < ndim; i++) { - if (stride * buf->itemsize != buf->strides[i] && - buf->shape[i] > 1) - { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { PyErr_SetString(PyExc_ValueError, "Buffer not fortran contiguous."); goto fail; @@ -270,8 +276,7 @@ } else if (c_or_f_flag & __Pyx_IS_C_CONTIG) { Py_ssize_t stride = 1; for (i = ndim - 1; i >- 1; i--) { - if (stride * buf->itemsize != buf->strides[i] && - buf->shape[i] > 1) { + if (unlikely(stride * buf->itemsize != buf->strides[i] && buf->shape[i] > 1)) { PyErr_SetString(PyExc_ValueError, "Buffer not C contiguous."); goto fail; @@ -318,7 +323,7 @@ } buf = &memview->view; - if (buf->ndim != ndim) { + if (unlikely(buf->ndim != ndim)) { PyErr_Format(PyExc_ValueError, "Buffer has wrong number of dimensions (expected %d, got %d)", ndim, buf->ndim); @@ -327,10 +332,10 @@ if (new_memview) { __Pyx_BufFmt_Init(&ctx, stack, dtype); - if (!__Pyx_BufFmt_CheckString(&ctx, buf->format)) goto fail; + if (unlikely(!__Pyx_BufFmt_CheckString(&ctx, buf->format))) goto fail; } - if ((unsigned) buf->itemsize != dtype->size) { + if (unlikely((unsigned) buf->itemsize != dtype->size)) { PyErr_Format(PyExc_ValueError, "Item size of buffer (%" CYTHON_FORMAT_SSIZE_T "u byte%s) " "does not match size of '%s' (%" CYTHON_FORMAT_SSIZE_T "u byte%s)", @@ -343,18 +348,22 @@ } /* Check axes */ - for (i = 0; i < ndim; i++) { - spec = axes_specs[i]; - if (!__pyx_check_strides(buf, i, ndim, spec)) - goto fail; - if (!__pyx_check_suboffsets(buf, i, ndim, spec)) + if (buf->len > 0) { + // 0-sized arrays do not undergo these checks since their strides are + // irrelevant and they are always both C- and F-contiguous. + for (i = 0; i < ndim; i++) { + spec = axes_specs[i]; + if (unlikely(!__pyx_check_strides(buf, i, ndim, spec))) + goto fail; + if (unlikely(!__pyx_check_suboffsets(buf, i, ndim, spec))) + goto fail; + } + + /* Check contiguity */ + if (unlikely(buf->strides && !__pyx_verify_contig(buf, ndim, c_or_f_flag))) goto fail; } - /* Check contiguity */ - if (buf->strides && !__pyx_verify_contig(buf, ndim, c_or_f_flag)) - goto fail; - /* Initialize */ if (unlikely(__Pyx_init_memviewslice(memview, ndim, memviewslice, new_memview != NULL) == -1)) { @@ -387,11 +396,7 @@ Py_buffer *buf = &memview->view; __Pyx_RefNannySetupContext("init_memviewslice", 0); - if (!buf) { - PyErr_SetString(PyExc_ValueError, - "buf is NULL."); - goto fail; - } else if (memviewslice->memview || memviewslice->data) { + if (unlikely(memviewslice->memview || memviewslice->data)) { PyErr_SetString(PyExc_ValueError, "memviewslice is already initialized!"); goto fail; @@ -437,8 +442,12 @@ return retval; } +#ifndef Py_NO_RETURN +// available since Py3.3 +#define Py_NO_RETURN +#endif -static CYTHON_INLINE void __pyx_fatalerror(const char *fmt, ...) { +static void __pyx_fatalerror(const char *fmt, ...) Py_NO_RETURN { va_list vargs; char msg[200]; @@ -447,11 +456,10 @@ #else va_start(vargs); #endif - vsnprintf(msg, 200, fmt, vargs); - Py_FatalError(msg); - va_end(vargs); + + Py_FatalError(msg); } static CYTHON_INLINE int @@ -480,47 +488,49 @@ static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int first_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; - if (!memview || (PyObject *) memview == Py_None) - return; /* allow uninitialized memoryview assignment */ - - if (__pyx_get_slice_count(memview) < 0) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - first_time = __pyx_add_acquisition_count(memview) == 0; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + // Allow uninitialized memoryview assignment and do not ref-count None. + return; + } - if (first_time) { - if (have_gil) { - Py_INCREF((PyObject *) memview); + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + // First acquisition => keep the memoryview object alive. + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } } else { - PyGILState_STATE _gilstate = PyGILState_Ensure(); - Py_INCREF((PyObject *) memview); - PyGILState_Release(_gilstate); + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } } -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int last_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; - if (!memview ) { - return; - } else if ((PyObject *) memview == Py_None) { + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + // Do not ref-count None. memslice->memview = NULL; return; } - if (__pyx_get_slice_count(memview) <= 0) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - last_time = __pyx_sub_acquisition_count(memview) == 1; + old_acquisition_count = __pyx_sub_acquisition_count(memview); memslice->data = NULL; - if (last_time) { + if (likely(old_acquisition_count > 1)) { + // Still other slices out there => we do not own the reference. + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + // Last slice => discard owned Python reference to memoryview object. if (have_gil) { Py_CLEAR(memslice->memview); } else { @@ -529,7 +539,8 @@ PyGILState_Release(_gilstate); } } else { - memslice->memview = NULL; + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } @@ -564,7 +575,7 @@ __Pyx_RefNannySetupContext("__pyx_memoryview_copy_new_contig", 0); for (i = 0; i < ndim; i++) { - if (from_mvs->suboffsets[i] >= 0) { + if (unlikely(from_mvs->suboffsets[i] >= 0)) { PyErr_Format(PyExc_ValueError, "Cannot copy memoryview slice with " "indirect dimensions (axis %d)", i); goto fail; @@ -689,29 +700,21 @@ } -////////// MemviewSliceIsCContig.proto ////////// - -#define __pyx_memviewslice_is_c_contig{{ndim}}(slice) \ - __pyx_memviewslice_is_contig(slice, 'C', {{ndim}}) - +////////// MemviewSliceCheckContig.proto ////////// -////////// MemviewSliceIsFContig.proto ////////// - -#define __pyx_memviewslice_is_f_contig{{ndim}}(slice) \ - __pyx_memviewslice_is_contig(slice, 'F', {{ndim}}) +#define __pyx_memviewslice_is_contig_{{contig_type}}{{ndim}}(slice) \ + __pyx_memviewslice_is_contig(slice, '{{contig_type}}', {{ndim}}) ////////// MemviewSliceIsContig.proto ////////// -static int __pyx_memviewslice_is_contig(const {{memviewslice_name}} mvs, - char order, int ndim); +static int __pyx_memviewslice_is_contig(const {{memviewslice_name}} mvs, char order, int ndim);/*proto*/ ////////// MemviewSliceIsContig ////////// static int -__pyx_memviewslice_is_contig(const {{memviewslice_name}} mvs, - char order, int ndim) +__pyx_memviewslice_is_contig(const {{memviewslice_name}} mvs, char order, int ndim) { int i, index, step, start; Py_ssize_t itemsize = mvs.memview->view.itemsize; @@ -775,7 +778,7 @@ {{if from_py_function}} static CYTHON_INLINE int {{set_function}}(const char *itemp, PyObject *obj) { {{dtype}} value = {{from_py_function}}(obj); - if ({{error_condition}}) + if (unlikely({{error_condition}})) return 0; *({{dtype}} *) itemp = value; return 1; @@ -850,28 +853,37 @@ { Py_ssize_t __pyx_tmp_idx = {{idx}}; - Py_ssize_t __pyx_tmp_shape = {{src}}.shape[{{dim}}]; + + {{if wraparound or boundscheck}} + Py_ssize_t __pyx_tmp_shape = {{src}}.shape[{{dim}}]; + {{endif}} + Py_ssize_t __pyx_tmp_stride = {{src}}.strides[{{dim}}]; - if ({{wraparound}} && (__pyx_tmp_idx < 0)) - __pyx_tmp_idx += __pyx_tmp_shape; + {{if wraparound}} + if (__pyx_tmp_idx < 0) + __pyx_tmp_idx += __pyx_tmp_shape; + {{endif}} + + {{if boundscheck}} + if (unlikely(!__Pyx_is_valid_index(__pyx_tmp_idx, __pyx_tmp_shape))) { + {{if not have_gil}} + #ifdef WITH_THREAD + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); + #endif + {{endif}} - if ({{boundscheck}} && (__pyx_tmp_idx < 0 || __pyx_tmp_idx >= __pyx_tmp_shape)) { - {{if not have_gil}} - #ifdef WITH_THREAD - PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); - #endif - {{endif}} - - PyErr_SetString(PyExc_IndexError, "Index out of bounds (axis {{dim}})"); - - {{if not have_gil}} - #ifdef WITH_THREAD - PyGILState_Release(__pyx_gilstate_save); - #endif - {{endif}} + PyErr_SetString(PyExc_IndexError, + "Index out of bounds (axis {{dim}})"); - {{error_goto}} - } + {{if not have_gil}} + #ifdef WITH_THREAD + PyGILState_Release(__pyx_gilstate_save); + #endif + {{endif}} + + {{error_goto}} + } + {{endif}} {{if all_dimensions_direct}} {{dst}}.data += __pyx_tmp_idx * __pyx_tmp_stride; @@ -917,7 +929,7 @@ ////////// FillStrided1DScalar ////////// /* Fill a slice with a scalar value. The dimension is direct and strided or contiguous */ -/* This can be used as a callback for the memoryview object to efficienty assign a scalar */ +/* This can be used as a callback for the memoryview object to efficiently assign a scalar */ /* Currently unused */ static void __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t stride, diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/MemoryView.pyx cython-0.20.1+1~202203241016-9537/Cython/Utility/MemoryView.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Utility/MemoryView.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/MemoryView.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,8 @@ #################### View.MemoryView #################### +# cython: language_level=3str +# cython: binding=False + # This utility provides cython.array and cython.view.memoryview from __future__ import absolute_import @@ -8,8 +11,12 @@ # from cpython cimport ... cdef extern from "Python.h": + ctypedef struct PyObject int PyIndex_Check(object) object PyLong_FromVoidPtr(void *) + PyObject *PyExc_IndexError + PyObject *PyExc_ValueError + PyObject *PyExc_MemoryError cdef extern from "pythread.h": ctypedef void *PyThread_type_lock @@ -49,7 +56,7 @@ Py_ssize_t suboffsets[{{max_dims}}] void __PYX_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) - void __PYX_XDEC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) ctypedef struct __pyx_buffer "Py_buffer": PyObject *obj @@ -64,15 +71,13 @@ PyBUF_WRITABLE PyBUF_STRIDES PyBUF_INDIRECT + PyBUF_ND PyBUF_RECORDS + PyBUF_RECORDS_RO ctypedef struct __Pyx_TypeInfo: pass - cdef object capsule "__pyx_capsule_create" (void *p, char *sig) - cdef int __pyx_array_getbuffer(PyObject *obj, Py_buffer view, int flags) - cdef int __pyx_memoryview_getbuffer(PyObject *obj, Py_buffer view, int flags) - cdef extern from *: ctypedef int __pyx_atomic_int {{memviewslice_name}} slice_copy_contig "__pyx_memoryview_copy_new_contig"( @@ -92,9 +97,6 @@ void free(void *) nogil void *memcpy(void *dest, void *src, size_t n) nogil - - - # ### cython.array class # @@ -121,17 +123,16 @@ mode="c", bint allocate_buffer=True): cdef int idx - cdef Py_ssize_t i, dim - cdef PyObject **p + cdef Py_ssize_t dim self.ndim = len(shape) self.itemsize = itemsize if not self.ndim: - raise ValueError("Empty shape tuple for cython.array") + raise ValueError, "Empty shape tuple for cython.array" if itemsize <= 0: - raise ValueError("itemsize <= 0 for cython.array") + raise ValueError, "itemsize <= 0 for cython.array" if not isinstance(format, bytes): format = format.encode('ASCII') @@ -143,76 +144,66 @@ self._strides = self._shape + self.ndim if not self._shape: - raise MemoryError("unable to allocate shape and strides.") + raise MemoryError, "unable to allocate shape and strides." # cdef Py_ssize_t dim, stride for idx, dim in enumerate(shape): if dim <= 0: - raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) + raise ValueError, f"Invalid shape in axis {idx}: {dim}." self._shape[idx] = dim cdef char order - if mode == 'fortran': - order = b'F' - self.mode = u'fortran' - elif mode == 'c': + if mode == 'c': order = b'C' self.mode = u'c' + elif mode == 'fortran': + order = b'F' + self.mode = u'fortran' else: - raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) + raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" - self.len = fill_contig_strides_array(self._shape, self._strides, - itemsize, self.ndim, order) + self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) self.free_data = allocate_buffer self.dtype_is_object = format == b'O' - if allocate_buffer: - # use malloc() for backwards compatibility - # in case external code wants to change the data pointer - self.data = malloc(self.len) - if not self.data: - raise MemoryError("unable to allocate array data.") - if self.dtype_is_object: - p = self.data - for i in range(self.len / itemsize): - p[i] = Py_None - Py_INCREF(Py_None) + if allocate_buffer: + _allocate_buffer(self) @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): cdef int bufmode = -1 - if self.mode == u"c": - bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - elif self.mode == u"fortran": - bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - if not (flags & bufmode): - raise ValueError("Can only create a buffer that is contiguous in memory.") + if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + if self.mode == u"c": + bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + elif self.mode == u"fortran": + bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + if not (flags & bufmode): + raise ValueError, "Can only create a buffer that is contiguous in memory." info.buf = self.data info.len = self.len - info.ndim = self.ndim - info.shape = self._shape - info.strides = self._strides - info.suboffsets = NULL - info.itemsize = self.itemsize - info.readonly = 0 - if flags & PyBUF_FORMAT: - info.format = self.format + if flags & PyBUF_STRIDES: + info.ndim = self.ndim + info.shape = self._shape + info.strides = self._strides else: - info.format = NULL + info.ndim = 1 + info.shape = &self.len if flags & PyBUF_ND else NULL + info.strides = NULL + info.suboffsets = NULL + info.itemsize = self.itemsize + info.readonly = 0 + info.format = self.format if flags & PyBUF_FORMAT else NULL info.obj = self - __pyx_getbuffer = capsule( &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") - def __dealloc__(array self): if self.callback_free_data != NULL: self.callback_free_data(self.data) - elif self.free_data: + elif self.free_data and self.data is not NULL: if self.dtype_is_object: - refcount_objects_in_slice(self.data, self._shape, - self._strides, self.ndim, False) + refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) free(self.data) PyObject_Free(self._shape) @@ -225,6 +216,8 @@ flags = PyBUF_ANY_CONTIGUOUS|PyBUF_FORMAT|PyBUF_WRITABLE return memoryview(self, flags, self.dtype_is_object) + def __len__(self): + return self._shape[0] def __getattr__(self, attr): return getattr(self.memview, attr) @@ -236,16 +229,35 @@ self.memview[item] = value +@cname("__pyx_array_allocate_buffer") +cdef int _allocate_buffer(array self) except -1: + # use malloc() for backwards compatibility + # in case external code wants to change the data pointer + cdef Py_ssize_t i + cdef PyObject **p + + self.free_data = True + self.data = malloc(self.len) + if not self.data: + raise MemoryError, "unable to allocate array data." + + if self.dtype_is_object: + p = self.data + for i in range(self.len // self.itemsize): + p[i] = Py_None + Py_INCREF(Py_None) + return 0 + + @cname("__pyx_array_new") -cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, - char *mode, char *buf): +cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *c_mode, char *buf): cdef array result + cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. - if buf == NULL: - result = array(shape, itemsize, format, mode.decode('ASCII')) + if buf is NULL: + result = array.__new__(array, shape, itemsize, format, mode) else: - result = array(shape, itemsize, format, mode.decode('ASCII'), - allocate_buffer=False) + result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) result.data = buf return result @@ -323,7 +335,7 @@ @cname('__pyx_memoryview') -cdef class memoryview(object): +cdef class memoryview: cdef object obj cdef object _size @@ -368,6 +380,10 @@ def __dealloc__(memoryview self): if self.obj is not None: __Pyx_ReleaseBuffer(&self.view) + elif (<__pyx_buffer *> &self.view).obj == Py_None: + # Undo the incref in __cinit__() above. + (<__pyx_buffer *> &self.view).obj = NULL + Py_DECREF(Py_None) cdef int i global __pyx_memoryview_thread_locks_used @@ -406,6 +422,9 @@ return self.convert_item_to_object(itemp) def __setitem__(memoryview self, object index, object value): + if self.view.readonly: + raise TypeError, "Cannot assign to read-only memoryview" + have_slices, index = _unellipsify(index, self.view.ndim) if have_slices: @@ -420,7 +439,7 @@ cdef is_slice(self, obj): if not isinstance(obj, memoryview): try: - obj = memoryview(obj, self.flags|PyBUF_ANY_CONTIGUOUS, + obj = memoryview(obj, self.flags & ~PyBUF_WRITABLE | PyBUF_ANY_CONTIGUOUS, self.dtype_is_object) except TypeError: return None @@ -430,10 +449,10 @@ cdef setitem_slice_assignment(self, dst, src): cdef {{memviewslice_name}} dst_slice cdef {{memviewslice_name}} src_slice + cdef {{memviewslice_name}} msrc = get_slice_from_memview(src, &src_slice)[0] + cdef {{memviewslice_name}} mdst = get_slice_from_memview(dst, &dst_slice)[0] - memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], - get_slice_from_memview(dst, &dst_slice)[0], - src.ndim, dst.ndim, self.dtype_is_object) + memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) cdef setitem_slice_assign_scalar(self, memoryview dst, value): cdef int array[128] @@ -481,7 +500,7 @@ try: result = struct.unpack(self.view.format, bytesitem) except struct.error: - raise ValueError("Unable to convert item to object") + raise ValueError, "Unable to convert item to object" else: if len(self.view.format) == 1: return result[0] @@ -505,7 +524,10 @@ @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): - if flags & PyBUF_STRIDES: + if flags & PyBUF_WRITABLE and self.view.readonly: + raise ValueError, "Cannot create writable memory view from read-only memoryview" + + if flags & PyBUF_ND: info.shape = self.view.shape else: info.shape = NULL @@ -529,12 +551,10 @@ info.ndim = self.view.ndim info.itemsize = self.view.itemsize info.len = self.view.len - info.readonly = 0 + info.readonly = self.view.readonly info.obj = self - __pyx_getbuffer = capsule( &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") - - # Some properties that have the same sematics as in NumPy + # Some properties that have the same semantics as in NumPy @property def T(self): cdef _memoryviewslice result = memoryview_copy(self) @@ -543,6 +563,9 @@ @property def base(self): + return self._get_base() + + cdef _get_base(self): return self.obj @property @@ -553,7 +576,7 @@ def strides(self): if self.view.strides == NULL: # Note: we always ask for strides, so if this is not set it's a bug - raise ValueError("Buffer view does not expose strides") + raise ValueError, "Buffer view does not expose strides" return tuple([stride for stride in self.view.strides[:self.view.ndim]]) @@ -654,39 +677,34 @@ Replace all ellipses with full slices and fill incomplete indices with full slices. """ - if not isinstance(index, tuple): - tup = (index,) - else: - tup = index + cdef Py_ssize_t idx + tup = index if isinstance(index, tuple) else (index,) - result = [] + result = [slice(None)] * ndim have_slices = False seen_ellipsis = False - for idx, item in enumerate(tup): + idx = 0 + for item in tup: if item is Ellipsis: if not seen_ellipsis: - result.extend([slice(None)] * (ndim - len(tup) + 1)) + idx += ndim - len(tup) seen_ellipsis = True - else: - result.append(slice(None)) have_slices = True else: - if not isinstance(item, slice) and not PyIndex_Check(item): - raise TypeError("Cannot index with type '%s'" % type(item)) - - have_slices = have_slices or isinstance(item, slice) - result.append(item) - - nslices = ndim - len(result) - if nslices: - result.extend([slice(None)] * nslices) + if isinstance(item, slice): + have_slices = True + elif not PyIndex_Check(item): + raise TypeError, f"Cannot index with type '{type(item)}'" + result[idx] = item + idx += 1 + nslices = ndim - idx return have_slices or nslices, tuple(result) cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): for suboffset in suboffsets[:ndim]: if suboffset >= 0: - raise ValueError("Indirect dimensions not supported") + raise ValueError, "Indirect dimensions not supported" # ### Slicing a memoryview @@ -726,15 +744,16 @@ # may not be as expected" cdef {{memviewslice_name}} *p_dst = &dst cdef int *p_suboffset_dim = &suboffset_dim - cdef Py_ssize_t start, stop, step + cdef Py_ssize_t start, stop, step, cindex cdef bint have_start, have_stop, have_step for dim, index in enumerate(indices): if PyIndex_Check(index): + cindex = index slice_memviewslice( p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], dim, new_ndim, p_suboffset_dim, - index, 0, 0, # start, stop, step + cindex, 0, 0, # start, stop, step 0, 0, 0, # have_{start,stop,step} False) elif index is None: @@ -815,13 +834,16 @@ if start < 0: start += shape if not 0 <= start < shape: - _err_dim(IndexError, "Index out of bounds (axis %d)", dim) + _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) else: # index is a slice - negative_step = have_step != 0 and step < 0 - - if have_step and step == 0: - _err_dim(ValueError, "Step may not be zero (axis %d)", dim) + if have_step: + negative_step = step < 0 + if step == 0: + _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + else: + negative_step = False + step = 1 # check our bounds and set defaults if have_start: @@ -853,9 +875,6 @@ else: stop = shape - if not have_step: - step = 1 - # len = ceil( (stop - start) / step ) with cython.cdivision(True): new_shape = (stop - start) // step @@ -871,7 +890,7 @@ dst.shape[new_ndim] = new_shape dst.suboffsets[new_ndim] = suboffset - # Add the slicing or idexing offsets to the right suboffset or base data * + # Add the slicing or indexing offsets to the right suboffset or base data * if suboffset_dim[0] < 0: dst.data += start * stride else: @@ -882,7 +901,7 @@ if new_ndim == 0: dst.data = ( dst.data)[0] + suboffset else: - _err_dim(IndexError, "All dimensions preceding dimension %d " + _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " "must be indexed and not sliced", dim) else: suboffset_dim[0] = new_ndim @@ -900,7 +919,7 @@ cdef char *resultp if view.ndim == 0: - shape = view.len / itemsize + shape = view.len // itemsize stride = itemsize else: shape = view.shape[dim] @@ -911,10 +930,10 @@ if index < 0: index += view.shape[dim] if index < 0: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" if index >= shape: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" resultp = bufp + index * stride if suboffset >= 0: @@ -926,7 +945,7 @@ ### Transposing a memoryviewslice # @cname('__pyx_memslice_transpose') -cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: +cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except -1: cdef int ndim = memslice.memview.view.ndim cdef Py_ssize_t *shape = memslice.shape @@ -934,15 +953,15 @@ # reverse strides and shape cdef int i, j - for i in range(ndim / 2): + for i in range(ndim // 2): j = ndim - 1 - i strides[i], strides[j] = strides[j], strides[i] shape[i], shape[j] = shape[j], shape[i] if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: - _err(ValueError, "Cannot transpose memoryview with indirect dimensions") + _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") - return 1 + return 0 # ### Creating new memoryview objects from slices and memoryviews @@ -960,7 +979,7 @@ cdef int (*to_dtype_func)(char *, object) except 0 def __dealloc__(self): - __PYX_XDEC_MEMVIEW(&self.from_slice, 1) + __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) cdef convert_item_to_object(self, char *itemp): if self.to_object_func != NULL: @@ -974,12 +993,9 @@ else: memoryview.assign_item_from_object(self, itemp, value) - @property - def base(self): + cdef _get_base(self): return self.from_object - __pyx_getbuffer = capsule( &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") - @cname('__pyx_memoryview_fromslice') cdef memoryview_fromslice({{memviewslice_name}} memviewslice, @@ -996,12 +1012,12 @@ # assert 0 < ndim <= memviewslice.memview.view.ndim, ( # ndim, memviewslice.memview.view.ndim) - result = _memoryviewslice(None, 0, dtype_is_object) + result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) result.from_slice = memviewslice __PYX_INC_MEMVIEW(&memviewslice, 1) - result.from_object = ( memviewslice.memview).base + result.from_object = ( memviewslice.memview)._get_base() result.typeinfo = memviewslice.memview.typeinfo result.view = memviewslice.memview.view @@ -1010,7 +1026,10 @@ (<__pyx_buffer *> &result.view).obj = Py_None Py_INCREF(Py_None) - result.flags = PyBUF_RECORDS + if (memviewslice.memview).flags & PyBUF_WRITABLE: + result.flags = PyBUF_RECORDS + else: + result.flags = PyBUF_RECORDS_RO result.view.shape = result.from_slice.shape result.view.strides = result.from_slice.strides @@ -1033,7 +1052,7 @@ @cname('__pyx_memoryview_get_slice_from_memoryview') cdef {{memviewslice_name}} *get_slice_from_memview(memoryview memview, - {{memviewslice_name}} *mslice): + {{memviewslice_name}} *mslice) except NULL: cdef _memoryviewslice obj if isinstance(memview, _memoryviewslice): obj = memview @@ -1090,10 +1109,7 @@ ### Copy the contents of a memoryview slices # cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: - if arg < 0: - return -arg - else: - return arg + return -arg if arg < 0 else arg @cname('__pyx_get_best_slice_order') cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) nogil: @@ -1159,11 +1175,10 @@ @cname('__pyx_memoryview_slice_get_size') cdef Py_ssize_t slice_get_size({{memviewslice_name}} *src, int ndim) nogil: "Return the size of the memory occupied by the slice in number of bytes" - cdef int i - cdef Py_ssize_t size = src.memview.view.itemsize + cdef Py_ssize_t shape, size = src.memview.view.itemsize - for i in range(ndim): - size *= src.shape[i] + for shape in src.shape[:ndim]: + size *= shape return size @@ -1180,11 +1195,11 @@ if order == 'F': for idx in range(ndim): strides[idx] = stride - stride = stride * shape[idx] + stride *= shape[idx] else: for idx in range(ndim - 1, -1, -1): strides[idx] = stride - stride = stride * shape[idx] + stride *= shape[idx] return stride @@ -1205,7 +1220,7 @@ result = malloc(size) if not result: - _err(MemoryError, NULL) + _err_no_memory() # tmpslice[0] = src tmpslice.data = result @@ -1214,8 +1229,7 @@ tmpslice.shape[i] = src.shape[i] tmpslice.suboffsets[i] = -1 - fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, - ndim, order) + fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # We need to broadcast strides again for i in range(ndim): @@ -1234,19 +1248,20 @@ @cname('__pyx_memoryview_err_extents') cdef int _err_extents(int i, Py_ssize_t extent1, Py_ssize_t extent2) except -1 with gil: - raise ValueError("got differing extents in dimension %d (got %d and %d)" % - (i, extent1, extent2)) + raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" @cname('__pyx_memoryview_err_dim') -cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: - raise error(msg.decode('ascii') % dim) +cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + raise error, msg % dim @cname('__pyx_memoryview_err') -cdef int _err(object error, char *msg) except -1 with gil: - if msg != NULL: - raise error(msg.decode('ascii')) - else: - raise error +cdef int _err(PyObject *error, str msg) except -1 with gil: + raise error, msg + +@cname('__pyx_memoryview_err_no_memory') +cdef int _err_no_memory() except -1 with gil: + raise MemoryError + @cname('__pyx_memoryview_copy_contents') cdef int memoryview_copy_contents({{memviewslice_name}} src, @@ -1281,7 +1296,7 @@ _err_extents(i, dst.shape[i], src.shape[i]) if src.suboffsets[i] >= 0: - _err_dim(ValueError, "Dimension %d is not direct", i) + _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) if slices_overlap(&src, &dst, ndim, itemsize): # slices overlap, copy to temp, copy temp to dst @@ -1301,9 +1316,9 @@ if direct_copy: # Contiguous slices with same order - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) memcpy(dst.data, src.data, slice_get_size(&src, ndim)) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1313,9 +1328,9 @@ transpose_memslice(&src) transpose_memslice(&dst) - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) copy_strided_to_strided(&src, &dst, ndim, itemsize) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1338,18 +1353,15 @@ mslice.suboffsets[i] = -1 # -### Take care of refcounting the objects in slices. Do this seperately from any copying, +### Take care of refcounting the objects in slices. Do this separately from any copying, ### to minimize acquiring the GIL # @cname('__pyx_memoryview_refcount_copying') -cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, - int ndim, bint inc) nogil: - # incref or decref the objects in the destination slice if the dtype is - # object +cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, int ndim, bint inc) nogil: + # incref or decref the objects in the destination slice if the dtype is object if dtype_is_object: - refcount_objects_in_slice_with_gil(dst.data, dst.shape, - dst.strides, ndim, inc) + refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, @@ -1361,6 +1373,7 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, bint inc): cdef Py_ssize_t i + cdef Py_ssize_t stride = strides[0] for i in range(shape[0]): if ndim == 1: @@ -1369,10 +1382,9 @@ else: Py_DECREF(( data)[0]) else: - refcount_objects_in_slice(data, shape + 1, strides + 1, - ndim - 1, inc) + refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) - data += strides[0] + data += stride # ### Scalar to slice assignment @@ -1381,10 +1393,9 @@ cdef void slice_assign_scalar({{memviewslice_name}} *dst, int ndim, size_t itemsize, void *item, bint dtype_is_object) nogil: - refcount_copying(dst, dtype_is_object, ndim, False) - _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, - itemsize, item) - refcount_copying(dst, dtype_is_object, ndim, True) + refcount_copying(dst, dtype_is_object, ndim, inc=False) + _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + refcount_copying(dst, dtype_is_object, ndim, inc=True) @cname('__pyx_memoryview__slice_assign_scalar') @@ -1401,8 +1412,7 @@ data += stride else: for i in range(extent): - _slice_assign_scalar(data, shape + 1, strides + 1, - ndim - 1, itemsize, item) + _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) data += stride @@ -1448,9 +1458,11 @@ cdef __Pyx_StructField *field cdef __pyx_typeinfo_string fmt cdef bytes part, result + cdef Py_ssize_t i if type.typegroup == 'S': - assert type.fields != NULL and type.fields.type != NULL + assert type.fields != NULL + assert type.fields.type != NULL if type.flags & __PYX_BUF_FLAGS_PACKED_STRUCT: alignment = b'^' @@ -1468,10 +1480,9 @@ result = alignment.join(parts) + b'}' else: fmt = __Pyx_TypeInfoToFormat(type) + result = fmt.string if type.arraysize[0]: - extents = [unicode(type.arraysize[i]) for i in range(type.ndim)] - result = (u"(%s)" % u','.join(extents)).encode('ascii') + fmt.string - else: - result = fmt.string + extents = [f"{type.arraysize[i]}" for i in range(type.ndim)] + result = f"({u','.join(extents)})".encode('ascii') + result return result diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/ModuleSetupCode.c cython-0.20.1+1~202203241016-9537/Cython/Utility/ModuleSetupCode.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/ModuleSetupCode.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/ModuleSetupCode.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,3 +1,16 @@ +/////////////// InitLimitedAPI /////////////// + +#if defined(CYTHON_LIMITED_API) && 0 /* disabled: enabling Py_LIMITED_API needs more work */ + #ifndef Py_LIMITED_API + #if CYTHON_LIMITED_API+0 > 0x03030000 + #define Py_LIMITED_API CYTHON_LIMITED_API + #else + #define Py_LIMITED_API 0x03030000 + #endif + #endif +#endif + + /////////////// CModulePreamble /////////////// #include /* For offsetof */ @@ -5,7 +18,7 @@ #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif -#if !defined(WIN32) && !defined(MS_WINDOWS) +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif @@ -24,11 +37,12 @@ #define DL_EXPORT(t) t #endif +// For use in DL_IMPORT/DL_EXPORT macros. +#define __PYX_COMMA , + #ifndef HAVE_LONG_LONG // CPython has required PY_LONG_LONG support for years, even if HAVE_LONG_LONG is not defined for us - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) - #define HAVE_LONG_LONG - #endif + #define HAVE_LONG_LONG #endif #ifndef PY_LONG_LONG @@ -39,15 +53,27 @@ #define Py_HUGE_VAL HUGE_VAL #endif -#ifdef PYPY_VERSION - #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + // GRAALVM_PYTHON test comes before PyPy test because GraalPython unhelpfully defines PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 #undef CYTHON_USE_UNICODE_INTERNALS @@ -64,60 +90,157 @@ #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 -#elif defined(PYSTON_VERSION) + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 +#elif defined(CYTHON_LIMITED_API) + // EXPERIMENTAL !! #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif + // CYTHON_CLINE_IN_TRACEBACK is currently disabled for the Limited API + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 #undef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 0 #ifndef CYTHON_AVOID_BORROWED_REFS #define CYTHON_AVOID_BORROWED_REFS 0 #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 - + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 1 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 #else #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif #if PY_MAJOR_VERSION < 3 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #elif !defined(CYTHON_USE_ASYNC_SLOTS) #define CYTHON_USE_ASYNC_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #ifndef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 1 #endif #ifndef CYTHON_USE_PYLIST_INTERNALS @@ -126,7 +249,9 @@ #ifndef CYTHON_USE_UNICODE_INTERNALS #define CYTHON_USE_UNICODE_INTERNALS 1 #endif - #if PY_VERSION_HEX < 0x030300F0 + #if PY_VERSION_HEX < 0x030300F0 || PY_VERSION_HEX >= 0x030B00A2 + // Python 3.11a2 hid _PyLong_FormatAdvancedWriter and _PyFloat_FormatAdvancedWriter + // therefore disable unicode writer until a better alternative appears #undef CYTHON_USE_UNICODE_WRITER #define CYTHON_USE_UNICODE_WRITER 0 #elif !defined(CYTHON_USE_UNICODE_WRITER) @@ -144,23 +269,252 @@ #ifndef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 1 #endif + #ifndef CYTHON_FAST_GIL + // Py3<3.5.2 does not support _PyThreadState_UncheckedGet(). + #define CYTHON_FAST_GIL (PY_MAJOR_VERSION < 3 || PY_VERSION_HEX >= 0x03060000) + #endif + #ifndef CYTHON_METH_FASTCALL + // CPython 3.6 introduced METH_FASTCALL but with slightly different + // semantics. It became stable starting from CPython 3.7. + #define CYTHON_METH_FASTCALL (PY_VERSION_HEX >= 0x030700A1) + #endif #ifndef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 1 #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + // EXPERIMENTAL !! + #define CYTHON_USE_MODULE_STATE 0 + #endif + #if PY_VERSION_HEX < 0x030400a1 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #elif !defined(CYTHON_USE_TP_FINALIZE) + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #if PY_VERSION_HEX < 0x030600B1 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS 1 + #endif + #if PY_VERSION_HEX < 0x030700A3 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #elif !defined(CYTHON_USE_EXC_INFO_STACK) + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif #endif #if !defined(CYTHON_FAST_PYCCALL) #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif +#if !defined(CYTHON_VECTORCALL) +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif + +/* Whether to use METH_FASTCALL with a fake backported implementation of vectorcall */ +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) + #if CYTHON_USE_PYLONG_INTERNALS - #include "longintrepr.h" + #if PY_MAJOR_VERSION < 3 + #include "longintrepr.h" + #endif /* These short defines can easily conflict with other code */ #undef SHIFT #undef BASE #undef MASK + /* Compile-time sanity check that these are indeed equal. Github issue #2670. */ + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif + +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif + +// restrict +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif + +// unused attribute +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif + +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif + +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif + +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif + +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif + + +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) && __cplusplus >= 201103L + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #elif __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + + #if defined(__clang__ ) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 /* Xcode < 7.0 */ + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif + +/////////////// CInitCode /////////////// + +// inline attribute +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) + #define CYTHON_INLINE __inline__ + #elif defined(_MSC_VER) + #define CYTHON_INLINE __inline + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_INLINE inline + #else + #define CYTHON_INLINE + #endif +#endif + + +/////////////// CppInitCode /////////////// + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." #endif +// inline attribute +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif + +// Work around clang bug https://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} + +// Used for temporary variables of "reference" type. +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + // __Pyx_FakeReference(T& ref) : ptr(&ref) { } + // Const version needed as Cython doesn't know about const overloads (e.g. for stl containers). + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + // TODO(robertwb): Delegate all operators (or auto-generate unwrapping code where needed). + template bool operator ==(U other) { return *ptr == other; } + template bool operator !=(U other) { return *ptr != other; } + private: + T *ptr; +}; + + +/////////////// PythonCompatibility /////////////// + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) #define Py_OptimizeFlag 0 #endif @@ -170,14 +524,96 @@ #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyType_Type +#if PY_VERSION_HEX >= 0x030B00A1 + static CYTHON_INLINE PyCodeObject* __Pyx_PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + // TODO - currently written to be simple and work in limited API etc. + // A more optimized version would be good + PyObject *kwds=NULL, *argcount=NULL, *posonlyargcount=NULL, *kwonlyargcount=NULL; + PyObject *nlocals=NULL, *stacksize=NULL, *flags=NULL, *replace=NULL, *call_result=NULL, *empty=NULL; + const char *fn_cstr=NULL; + const char *name_cstr=NULL; + PyCodeObject* co=NULL; + PyObject *type, *value, *traceback; + + // we must be able to call this while an exception is happening - thus clear then restore the state + PyErr_Fetch(&type, &value, &traceback); + + if (!(kwds=PyDict_New())) goto end; + if (!(argcount=PyLong_FromLong(a))) goto end; + if (PyDict_SetItemString(kwds, "co_argcount", argcount) != 0) goto end; + if (!(posonlyargcount=PyLong_FromLong(p))) goto end; + if (PyDict_SetItemString(kwds, "co_posonlyargcount", posonlyargcount) != 0) goto end; + if (!(kwonlyargcount=PyLong_FromLong(k))) goto end; + if (PyDict_SetItemString(kwds, "co_kwonlyargcount", kwonlyargcount) != 0) goto end; + if (!(nlocals=PyLong_FromLong(l))) goto end; + if (PyDict_SetItemString(kwds, "co_nlocals", nlocals) != 0) goto end; + if (!(stacksize=PyLong_FromLong(s))) goto end; + if (PyDict_SetItemString(kwds, "co_stacksize", stacksize) != 0) goto end; + if (!(flags=PyLong_FromLong(f))) goto end; + if (PyDict_SetItemString(kwds, "co_flags", flags) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_code", code) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_consts", c) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_names", n) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_varnames", v) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_freevars", fv) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_cellvars", cell) != 0) goto end; + if (PyDict_SetItemString(kwds, "co_linetable", lnos) != 0) goto end; + + if (!(fn_cstr=PyUnicode_AsUTF8AndSize(fn, NULL))) goto end; + if (!(name_cstr=PyUnicode_AsUTF8AndSize(name, NULL))) goto end; + if (!(co = PyCode_NewEmpty(fn_cstr, name_cstr, fline))) goto end; + + if (!(replace = PyObject_GetAttrString((PyObject*)co, "replace"))) goto cleanup_code_too; + if (!(empty = PyTuple_New(0))) goto cleanup_code_too; // unfortunately __pyx_empty_tuple isn't available here + if (!(call_result = PyObject_Call(replace, empty, kwds))) goto cleanup_code_too; + + Py_XDECREF((PyObject*)co); + co = (PyCodeObject*)call_result; + call_result = NULL; + + if (0) { + cleanup_code_too: + Py_XDECREF((PyObject*)co); + co = NULL; + } + end: + Py_XDECREF(kwds); + Py_XDECREF(argcount); + Py_XDECREF(posonlyargcount); + Py_XDECREF(kwonlyargcount); + Py_XDECREF(nlocals); + Py_XDECREF(stacksize); + Py_XDECREF(replace); + Py_XDECREF(call_result); + Py_XDECREF(empty); + if (type) { + PyErr_Restore(type, value, traceback); + } + return co; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +#endif + +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) #endif #ifndef Py_TPFLAGS_CHECKTYPES @@ -193,34 +629,297 @@ #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif -#ifndef METH_FASTCALL - // new in CPython 3.6 - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); +#ifndef METH_STACKLESS + // already defined for Stackless Python (all versions) and C-Python >= 3.7 + // value if defined: Stackless Python < 3.6: 0x80 else 0x100 + #define METH_STACKLESS 0 +#endif +#if PY_VERSION_HEX <= 0x030700A3 || !defined(METH_FASTCALL) + // new in CPython 3.6, but changed in 3.7 - see + // positional-only parameters: + // https://bugs.python.org/issue29464 + // const args: + // https://bugs.python.org/issue32240 + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + // new in CPython 3.7, used to be old signature of _PyCFunctionFast() in 3.6 + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); #else #define __Pyx_PyCFunctionFast _PyCFunctionFast + #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords +#endif + +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif + +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif + +// PEP-573: PyCFunction holds reference to defining class (PyCMethodObject) +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif + +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x03060000 + //#elif PY_VERSION_HEX >= 0x03050200 + // Actually added in 3.5.2, but compiling against that does not guarantee that we get imported there. + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#elif PY_VERSION_HEX >= 0x03000000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_Current +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE void *__Pyx_PyModule_GetState(PyObject *op) +{ + void *result; + + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} #endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func) \ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) + +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE(obj), name, func_ctype) +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) #else -#define __Pyx_PyFastCFunction_Check(func) 0 + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) +#endif + +// TSS (Thread Specific Storage) API +#if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) +#include "pythread.h" +#define Py_tss_NEEDS_INIT 0 +typedef int Py_tss_t; +static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { + *key = PyThread_create_key(); + return 0; /* PyThread_create_key reports success always */ +} +static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { + Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); + *key = Py_tss_NEEDS_INIT; + return key; +} +static CYTHON_INLINE void PyThread_tss_free(Py_tss_t *key) { + PyObject_Free(key); +} +static CYTHON_INLINE int PyThread_tss_is_created(Py_tss_t *key) { + return *key != Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE void PyThread_tss_delete(Py_tss_t *key) { + PyThread_delete_key(*key); + *key = Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { + return PyThread_set_key_value(*key, value); +} +static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { + return PyThread_get_key_value(*key); +} +// PyThread_delete_key_value(key) is equivalent to PyThread_set_key_value(key, NULL) +// PyThread_ReInitTLS() is a no-op +#endif /* TSS (Thread Specific Storage) API */ + + +#if PY_MAJOR_VERSION < 3 + #if CYTHON_COMPILING_IN_PYPY + #if PYPY_VERSION_NUM < 0x07030600 + #if defined(__cplusplus) && __cplusplus >= 201402L + [[deprecated("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6")]] + #elif defined(__GNUC__) || defined(__clang__) + __attribute__ ((__deprecated__("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6"))) + #elif defined(_MSC_VER) + __declspec(deprecated("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6")) + #endif + static CYTHON_INLINE int PyGILState_Check(void) { + // PyGILState_Check is used to decide whether to release the GIL when we don't + // know that we have it. For PyPy2 it isn't possible to check. + // Therefore assume that we don't have the GIL (which causes us not to release it, + // but is "safe") + return 0; + } + #else // PYPY_VERSION_NUM < 0x07030600 + // PyPy2 >= 7.3.6 has PyGILState_Check + #endif // PYPY_VERSION_NUM < 0x07030600 + #else + // https://stackoverflow.com/a/25666624 + static CYTHON_INLINE int PyGILState_Check(void) { + PyThreadState * tstate = _PyThreadState_Current; + return tstate && (tstate == PyGILState_GetThisThreadState()); + } + #endif #endif -/* new Py3.3 unicode type (PEP 393) */ -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif + +#if PY_MAJOR_VERSION >= 3 || CYTHON_FUTURE_DIVISION + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B4 && CYTHON_USE_UNICODE_INTERNALS +// _PyDict_GetItem_KnownHash() exists since CPython 3.5, but it was +// dropping exceptions. Since 3.6, exceptions are kept. +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { + // This is tricky - we should return a borrowed reference but not swallow non-KeyError exceptions. 8-| + // But: this function is only used in Py2 and older PyPys, + // and currently only for argument parsing and other non-correctness-critical lookups + // and we know that 'name' is an interned 'str' with pre-calculated hash value (only comparisons can fail), + // thus, performance matters more than correctness here, especially in the "not found" case. +#if CYTHON_COMPILING_IN_PYPY + // So we ignore any exceptions in old PyPys ... + return PyDict_GetItem(dict, name); +#else + // and hack together a stripped-down and modified PyDict_GetItem() in CPython 2. + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); /* hash values of interned strings are always initialised */ + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + // error occurred + return NULL; + } + // found or not found + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif + +/* Type slots */ + +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) + #define __Pyx_PyObject_GetIterNextFunc(obj) (Py_TYPE(obj)->tp_iternext) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) + #define __Pyx_PyObject_GetIterNextFunc(obj) PyIter_Next +#endif + +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +// In Py3.8+, instances of heap types need to decref their type on deallocation. +// https://bugs.python.org/issue35810 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) { \ + PyTypeObject *type = Py_TYPE(obj); \ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)); \ + PyObject_GC_Del(obj); \ + Py_DECREF(type); \ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GetLength(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + // __Pyx_PyUnicode_DATA() and __Pyx_PyUnicode_READ() must go together, e.g. for iteration. + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + //#define __Pyx_PyUnicode_WRITE(k, d, i, ch) /* not available */ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 + + #if defined(PyUnicode_IS_READY) #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) + #else + // Py3.12 / PEP-623 will remove wstr type unicode strings and all of the PyUnicode_READY() machinery. + #define __Pyx_PyUnicode_READY(op) (0) + #endif + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #if defined(PyUnicode_IS_READY) && defined(PyUnicode_GET_SIZE) + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + // Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. + // https://www.python.org/dev/peps/pep-0623/ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #endif #else #define CYTHON_PEP393_ENABLED 0 #define PyUnicode_1BYTE_KIND 1 @@ -230,9 +929,9 @@ #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_KIND(u) ((int)sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - /* (void)(k) => avoid unused variable warning due to macro: */ + // (void)(k) => avoid unused variable warning due to macro: #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) @@ -247,35 +946,25 @@ PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) #endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif - -#if CYTHON_COMPILING_IN_PYSTON - // special C-API functions only in Pyston - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) || (PY_MAJOR_VERSION == 2 && PYPY_VERSION_NUM < 0x07030500) + #undef PyUnicode_Contains + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif #endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +// ("..." % x) must call PyNumber_Remainder() if x is a string subclass that implements "__rmod__()". +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyString_Check(b) && !PyString_CheckExact(b)))) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) @@ -293,6 +982,10 @@ #define PyString_Type PyUnicode_Type #define PyString_Check PyUnicode_Check #define PyString_CheckExact PyUnicode_CheckExact + // PyPy3 used to define "PyObject_Unicode" +#ifndef PyObject_Unicode + #define PyObject_Unicode PyObject_Str +#endif #endif #if PY_MAJOR_VERSION >= 3 @@ -303,11 +996,31 @@ #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) #endif +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj) \ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif + #ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) #endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif + +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) +#else + // NOTE: might fail with exception => check for -1 + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) +#endif #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject @@ -340,16 +1053,10 @@ #if PY_VERSION_HEX < 0x030200A4 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsHash_t #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsSsize_t #endif // backport of PyAsyncMethods from Py3.5 to older Py3.x versions @@ -359,88 +1066,228 @@ #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) #else + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef __Pyx_PyAsyncMethodsStruct typedef struct { unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL #endif -// restrict -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif + +/////////////// IncludeStructmemberH.proto /////////////// + +#include + + +/////////////// SmallCodeConfig.proto /////////////// + +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif #endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +/////////////// PyModInitFuncType.proto /////////////// -/////////////// CInitCode /////////////// +#ifndef CYTHON_NO_PYINIT_EXPORT +#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC -// inline attribute -#ifndef CYTHON_INLINE - #if defined(__GNUC__) - #define CYTHON_INLINE __inline__ - #elif defined(_MSC_VER) - #define CYTHON_INLINE __inline - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_INLINE inline - #else - #define CYTHON_INLINE - #endif +#elif PY_MAJOR_VERSION < 3 +// Py2: define this to void manually because PyMODINIT_FUNC adds __declspec(dllexport) to it's definition. +#ifdef __cplusplus +#define __Pyx_PyMODINIT_FUNC extern "C" void +#else +#define __Pyx_PyMODINIT_FUNC void #endif +#else +// Py3+: define this to PyObject * manually because PyMODINIT_FUNC adds __declspec(dllexport) to it's definition. +#ifdef __cplusplus +#define __Pyx_PyMODINIT_FUNC extern "C" PyObject * +#else +#define __Pyx_PyMODINIT_FUNC PyObject * +#endif +#endif -/////////////// CppInitCode /////////////// -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +/////////////// FastTypeChecks.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);/*proto*/ +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);/*proto*/ +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);/*proto*/ +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);/*proto*/ +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_Occurred(), err1, err2) + +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) + +/////////////// FastTypeChecks /////////////// +//@requires: Exceptions.c::PyThreadStateGet +//@requires: Exceptions.c::PyErrFetchRestore + +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} + +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + // should only get here for incompletely initialised types, i.e. never under normal usage patterns + return __Pyx_InBases(a, b); +} + +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + // should only get here for incompletely initialised types, i.e. never under normal usage patterns + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} + + +#if PY_MAJOR_VERSION == 2 +static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { + // PyObject_IsSubclass() can recurse and therefore is not safe + PyObject *exception, *value, *tb; + int res; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&exception, &value, &tb); + + res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; + // This function must not fail, so print the error here (which also clears it) + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + if (!res) { + res = PyObject_IsSubclass(err, exc_type2); + // This function must not fail, so print the error here (which also clears it) + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + } + + __Pyx_ErrRestore(exception, value, tb); + return res; +} +#else +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} #endif -// inline attribute -#ifndef CYTHON_INLINE - #define CYTHON_INLINE inline +// so far, we only call PyErr_GivenExceptionMatches() with an exception type (not instance) as first argument +// => optimise for that case + +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + // the tighter subtype checking in Py3 allows faster out-of-order comparison + for (i=0; i -void __Pyx_call_destructor(T& x) { - x.~T(); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { + if (likely(err == exc_type)) return 1; + if (likely(PyExceptionClass_Check(err))) { + if (likely(PyExceptionClass_Check(exc_type))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); + } else if (likely(PyTuple_Check(exc_type))) { + return __Pyx_PyErr_GivenExceptionMatchesTuple(err, exc_type); + } else { + // FIXME: Py3: PyErr_SetString(PyExc_TypeError, "catching classes that do not inherit from BaseException is not allowed"); + } + } + return PyErr_GivenExceptionMatches(err, exc_type); } -// Used for temporary variables of "reference" type. -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - // __Pyx_FakeReference(T& ref) : ptr(&ref) { } - // Const version needed as Cython doesn't know about const overloads (e.g. for stl containers). - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - operator T&() { return *ptr; } - // TODO(robertwb): Delegate all operators (or auto-generate unwrapping code where needed). - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; - private: - T *ptr; -}; +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { + // Only used internally with known exception types => pure safety check assertions. + assert(PyExceptionClass_Check(exc_type1)); + assert(PyExceptionClass_Check(exc_type2)); + if (likely(err == exc_type1 || err == exc_type2)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); + } + return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); +} + +#endif /////////////// MathInitCode /////////////// -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #if !defined(_USE_MATH_DEFINES) + #define _USE_MATH_DEFINES + #endif #endif #include @@ -466,33 +1313,11 @@ /////////////// UtilityFunctionPredeclarations.proto /////////////// -/* unused attribute */ -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif - -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif - typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; /*proto*/ /////////////// ForceInitThreads.proto /////////////// +//@proto_block: utility_code_proto_before_types #ifndef __PYX_FORCE_INIT_THREADS #define __PYX_FORCE_INIT_THREADS 0 @@ -500,12 +1325,107 @@ /////////////// InitThreads.init /////////////// -#ifdef WITH_THREAD +#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x030700F0 PyEval_InitThreads(); #endif + +/////////////// ModuleCreationPEP489 /////////////// +//@substitute: naming + +//#if CYTHON_PEP489_MULTI_PHASE_INIT +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + #if PY_VERSION_HEX >= 0x030700A1 + static PY_INT64_T main_interpreter_id = -1; + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return (unlikely(current_id == -1)) ? -1 : 0; + } else if (unlikely(main_interpreter_id != current_id)) + + #else + static PyInterpreterState *main_interpreter = NULL; + PyInterpreterState *current_interpreter = PyThreadState_Get()->interp; + if (!main_interpreter) { + main_interpreter = current_interpreter; + } else if (unlikely(main_interpreter != current_interpreter)) + #endif + + { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} + +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) +#else +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +#endif +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { +#if CYTHON_COMPILING_IN_LIMITED_API + result = PyModule_AddObject(module, to_name, value); +#else + result = PyDict_SetItemString(moddict, to_name, value); +#endif + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} + +static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + + // For now, we only have exactly one module instance. + if (__Pyx_check_single_interpreter()) + return NULL; + if (${module_cname}) + return __Pyx_NewRef(${module_cname}); + + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + +#if CYTHON_COMPILING_IN_LIMITED_API + moddict = module; +#else + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + // moddict is a borrowed reference +#endif + + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + + return module; +bad: + Py_XDECREF(module); + return NULL; +} +//#endif + + /////////////// CodeObjectCache.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API typedef struct { PyCodeObject* code_object; int code_line; @@ -522,11 +1442,13 @@ static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +#endif /////////////// CodeObjectCache /////////////// // Note that errors are simply ignored in the code below. // This is just a cache, if a lookup or insertion fails - so what? +#if !CYTHON_COMPILING_IN_LIMITED_API static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { @@ -592,7 +1514,7 @@ if (__pyx_code_cache.count == __pyx_code_cache.max_count) { int new_max = __pyx_code_cache.max_count + 64; entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( - __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + __pyx_code_cache.entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); if (unlikely(!entries)) { return; } @@ -607,9 +1529,11 @@ __pyx_code_cache.count++; Py_INCREF(code_object); } +#endif /////////////// CodeObjectCache.cleanup /////////////// + #if !CYTHON_COMPILING_IN_LIMITED_API if (__pyx_code_cache.entries) { __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; int i, count = __pyx_code_cache.count; @@ -621,6 +1545,7 @@ } PyMem_Free(entries); } + #endif /////////////// CheckBinaryVersion.proto /////////////// @@ -635,7 +1560,7 @@ if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { char message[200]; PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " + "compile time version %s of module '%.100s' " "does not match runtime version %s", ctversion, __Pyx_MODULE_NAME, rtversion); return PyErr_WarnEx(NULL, message, 1); @@ -643,6 +1568,22 @@ return 0; } +/////////////// IsLittleEndian.proto /////////////// + +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void); + +/////////////// IsLittleEndian /////////////// + +static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } S; + S.u32 = 0x01020304; + return S.u8[0] == 4; +} + /////////////// Refnanny.proto /////////////// #ifndef CYTHON_REFNANNY @@ -651,11 +1592,11 @@ #if CYTHON_REFNANNY typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; @@ -665,28 +1606,35 @@ #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ + } + #define __Pyx_RefNannyFinishContextNogil() { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __Pyx_RefNannyFinishContext(); \ + PyGILState_Release(__pyx_gilstate_save); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)) + #define __Pyx_RefNannyFinishContextNogil() __Pyx_RefNannyFinishContext() #endif #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) @@ -698,6 +1646,10 @@ #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ +#define __Pyx_Py_XDECREF_SET(r, v) do { \ + PyObject *tmp = (PyObject *) r; \ + r = v; Py_XDECREF(tmp); \ + } while (0) #define __Pyx_XDECREF_SET(r, v) do { \ PyObject *tmp = (PyObject *) r; \ r = v; __Pyx_XDECREF(tmp); \ @@ -716,9 +1668,9 @@ static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; void *r = NULL; - m = PyImport_ImportModule((char *)modname); + m = PyImport_ImportModule(modname); if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + p = PyObject_GetAttrString(m, "RefNannyAPI"); if (!p) goto end; r = PyLong_AsVoidPtr(p); end: @@ -728,18 +1680,37 @@ } #endif /* CYTHON_REFNANNY */ + +/////////////// ImportRefnannyAPI /////////////// + +#if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + + /////////////// RegisterModuleCleanup.proto /////////////// //@substitute: naming static void ${cleanup_cname}(PyObject *self); /*proto*/ + +#if PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY static int __Pyx_RegisterCleanup(void); /*proto*/ +#else +#define __Pyx_RegisterCleanup() (0) +#endif /////////////// RegisterModuleCleanup /////////////// //@substitute: naming -//@requires: ImportExport.c::ModuleImport -#if PY_MAJOR_VERSION < 3 -static PyObject* ${cleanup_cname}_atexit(PyObject *module, CYTHON_UNUSED PyObject *unused) { +#if PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY +static PyObject* ${cleanup_cname}_atexit(PyObject *module, PyObject *unused) { + CYTHON_UNUSED_VAR(unused); ${cleanup_cname}(module); Py_INCREF(Py_None); return Py_None; } @@ -766,7 +1737,7 @@ if (!cleanup_func) goto bad; - atexit = __Pyx_ImportModule("atexit"); + atexit = PyImport_ImportModule("atexit"); if (!atexit) goto bad; reg = PyObject_GetAttrString(atexit, "_exithandlers"); @@ -808,10 +1779,200 @@ Py_XDECREF(res); return ret; } +#endif + +/////////////// FastGil.init /////////////// +#ifdef WITH_THREAD +__Pyx_FastGilFuncInit(); +#endif + +/////////////// NoFastGil.proto /////////////// +//@proto_block: utility_code_proto_before_types + +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/////////////// FastGil.proto /////////////// +//@proto_block: utility_code_proto_before_types + +#if CYTHON_FAST_GIL + +struct __Pyx_FastGilVtab { + PyGILState_STATE (*Fast_PyGILState_Ensure)(void); + void (*Fast_PyGILState_Release)(PyGILState_STATE oldstate); + void (*FastGIL_Remember)(void); + void (*FastGIL_Forget)(void); +}; + +static void __Pyx_FastGIL_Noop(void) {} +static struct __Pyx_FastGilVtab __Pyx_FastGilFuncs = { + PyGILState_Ensure, + PyGILState_Release, + __Pyx_FastGIL_Noop, + __Pyx_FastGIL_Noop +}; + +static void __Pyx_FastGilFuncInit(void); + +#define __Pyx_PyGILState_Ensure __Pyx_FastGilFuncs.Fast_PyGILState_Ensure +#define __Pyx_PyGILState_Release __Pyx_FastGilFuncs.Fast_PyGILState_Release +#define __Pyx_FastGIL_Remember __Pyx_FastGilFuncs.FastGIL_Remember +#define __Pyx_FastGIL_Forget __Pyx_FastGilFuncs.FastGIL_Forget + +#ifdef WITH_THREAD + #ifndef CYTHON_THREAD_LOCAL + #if __STDC_VERSION__ >= 201112 + #define CYTHON_THREAD_LOCAL _Thread_local + #elif defined(__GNUC__) + #define CYTHON_THREAD_LOCAL __thread + #elif defined(_MSC_VER) + #define CYTHON_THREAD_LOCAL __declspec(thread) + #endif + #endif +#endif + #else -// fake call purely to work around "unused function" warning for __Pyx_ImportModule() -static int __Pyx_RegisterCleanup(void) { - if (0) __Pyx_ImportModule(NULL); - return 0; +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() +#endif + +/////////////// FastGil /////////////// +// The implementations of PyGILState_Ensure/Release calls PyThread_get_key_value +// several times which is turns out to be quite slow (slower in fact than +// acquiring the GIL itself). Simply storing it in a thread local for the +// common case is much faster. +// To make optimal use of this thread local, we attempt to share it between +// modules. + +#if CYTHON_FAST_GIL + +#define __Pyx_FastGIL_ABI_module __PYX_ABI_MODULE_NAME +#define __Pyx_FastGIL_PyCapsuleName "FastGilFuncs" +#define __Pyx_FastGIL_PyCapsule \ + __Pyx_FastGIL_ABI_module "." __Pyx_FastGIL_PyCapsuleName + +#ifdef CYTHON_THREAD_LOCAL + +#include "pythread.h" +#include "pystate.h" + +static CYTHON_THREAD_LOCAL PyThreadState *__Pyx_FastGil_tcur = NULL; +static CYTHON_THREAD_LOCAL int __Pyx_FastGil_tcur_depth = 0; +static int __Pyx_FastGil_autoTLSkey = -1; + +static CYTHON_INLINE void __Pyx_FastGIL_Remember0(void) { + ++__Pyx_FastGil_tcur_depth; +} + +static CYTHON_INLINE void __Pyx_FastGIL_Forget0(void) { + if (--__Pyx_FastGil_tcur_depth == 0) { + __Pyx_FastGil_tcur = NULL; + } +} + +static CYTHON_INLINE PyThreadState *__Pyx_FastGil_get_tcur(void) { + PyThreadState *tcur = __Pyx_FastGil_tcur; + if (tcur == NULL) { + tcur = __Pyx_FastGil_tcur = (PyThreadState*)PyThread_get_key_value(__Pyx_FastGil_autoTLSkey); + } + return tcur; +} + +static PyGILState_STATE __Pyx_FastGil_PyGILState_Ensure(void) { + int current; + PyThreadState *tcur; + __Pyx_FastGIL_Remember0(); + tcur = __Pyx_FastGil_get_tcur(); + if (tcur == NULL) { + // Uninitialized, need to initialize now. + return PyGILState_Ensure(); + } + current = tcur == __Pyx_PyThreadState_Current; + if (current == 0) { + PyEval_RestoreThread(tcur); + } + ++tcur->gilstate_counter; + return current ? PyGILState_LOCKED : PyGILState_UNLOCKED; +} + +static void __Pyx_FastGil_PyGILState_Release(PyGILState_STATE oldstate) { + PyThreadState *tcur = __Pyx_FastGil_get_tcur(); + __Pyx_FastGIL_Forget0(); + if (tcur->gilstate_counter == 1) { + // This is the last lock, do all the cleanup as well. + PyGILState_Release(oldstate); + } else { + --tcur->gilstate_counter; + if (oldstate == PyGILState_UNLOCKED) { + PyEval_SaveThread(); + } + } +} + +static void __Pyx_FastGilFuncInit0(void) { + /* Try to detect autoTLSkey. */ + int key; + void* this_thread_state = (void*) PyGILState_GetThisThreadState(); + for (key = 0; key < 100; key++) { + if (PyThread_get_key_value(key) == this_thread_state) { + __Pyx_FastGil_autoTLSkey = key; + break; + } + } + if (__Pyx_FastGil_autoTLSkey != -1) { + PyObject* capsule = NULL; + PyObject* abi_module = NULL; + __Pyx_PyGILState_Ensure = __Pyx_FastGil_PyGILState_Ensure; + __Pyx_PyGILState_Release = __Pyx_FastGil_PyGILState_Release; + __Pyx_FastGIL_Remember = __Pyx_FastGIL_Remember0; + __Pyx_FastGIL_Forget = __Pyx_FastGIL_Forget0; + capsule = PyCapsule_New(&__Pyx_FastGilFuncs, __Pyx_FastGIL_PyCapsule, NULL); + abi_module = PyImport_AddModule(__Pyx_FastGIL_ABI_module); + if (capsule && abi_module) { + PyObject_SetAttrString(abi_module, __Pyx_FastGIL_PyCapsuleName, capsule); + } + Py_XDECREF(capsule); + } } + +#else + +static void __Pyx_FastGilFuncInit0(void) { +} + +#endif + +static void __Pyx_FastGilFuncInit(void) { + struct __Pyx_FastGilVtab* shared = (struct __Pyx_FastGilVtab*)PyCapsule_Import(__Pyx_FastGIL_PyCapsule, 1); + if (shared) { + __Pyx_FastGilFuncs = *shared; + } else { + PyErr_Clear(); + __Pyx_FastGilFuncInit0(); + } +} + +#endif + +///////////////////// UtilityCodePragmas ///////////////////////// + +#if _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + +///////////////////// UtilityCodePragmasEnd ////////////////////// + +#if _MSV_VER +#pragma warning( pop ) /* undo whatever Cython has done to warnings */ #endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/NumpyImportArray.c cython-0.20.1+1~202203241016-9537/Cython/Utility/NumpyImportArray.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/NumpyImportArray.c 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/NumpyImportArray.c 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,21 @@ +///////////////////////// NumpyImportArray.init //////////////////// + +// comment below is deliberately kept in the generated C file to +// help users debug where this came from: +/* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * numpy._import_array + */ +#ifdef NPY_FEATURE_VERSION /* This is a public define that makes us reasonably confident it's "real" Numpy */ +// NO_IMPORT_ARRAY is Numpy's mechanism for indicating that import_array is handled elsewhere +#if !NO_IMPORT_ARRAY /* https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#c.NO_IMPORT_ARRAY */ +if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use 'numpy._import_array' to disable if you are certain you don't need it)."); + {{ err_goto }}; +} +#endif +#endif diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/ObjectHandling.c cython-0.20.1+1~202203241016-9537/Cython/Utility/ObjectHandling.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/ObjectHandling.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/ObjectHandling.c 2022-03-24 10:16:46.000000000 +0000 @@ -72,57 +72,76 @@ Py_DECREF(retval); __Pyx_RaiseTooManyValuesError(expected); return -1; - } else { - return __Pyx_IterFinish(); } - return 0; + + return __Pyx_IterFinish(); } /////////////// UnpackTuple2.proto /////////////// -static CYTHON_INLINE int __Pyx_unpack_tuple2(PyObject* tuple, PyObject** value1, PyObject** value2, - int is_tuple, int has_known_size, int decref_tuple); +#define __Pyx_unpack_tuple2(tuple, value1, value2, is_tuple, has_known_size, decref_tuple) \ + (likely(is_tuple || PyTuple_Check(tuple)) ? \ + (likely(has_known_size || PyTuple_GET_SIZE(tuple) == 2) ? \ + __Pyx_unpack_tuple2_exact(tuple, value1, value2, decref_tuple) : \ + (__Pyx_UnpackTupleError(tuple, 2), -1)) : \ + __Pyx_unpack_tuple2_generic(tuple, value1, value2, has_known_size, decref_tuple)) + +static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( + PyObject* tuple, PyObject** value1, PyObject** value2, int decref_tuple); +static int __Pyx_unpack_tuple2_generic( + PyObject* tuple, PyObject** value1, PyObject** value2, int has_known_size, int decref_tuple); /////////////// UnpackTuple2 /////////////// //@requires: UnpackItemEndCheck //@requires: UnpackTupleError //@requires: RaiseNeedMoreValuesToUnpack -static CYTHON_INLINE int __Pyx_unpack_tuple2(PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, - int is_tuple, int has_known_size, int decref_tuple) { - Py_ssize_t index; - PyObject *value1 = NULL, *value2 = NULL, *iter = NULL; - if (!is_tuple && unlikely(!PyTuple_Check(tuple))) { - iternextfunc iternext; - iter = PyObject_GetIter(tuple); - if (unlikely(!iter)) goto bad; - if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } - iternext = Py_TYPE(iter)->tp_iternext; - value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } - value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } - if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; - Py_DECREF(iter); - } else { - if (!has_known_size && unlikely(PyTuple_GET_SIZE(tuple) != 2)) { - __Pyx_UnpackTupleError(tuple, 2); - goto bad; - } +static CYTHON_INLINE int __Pyx_unpack_tuple2_exact( + PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, int decref_tuple) { + PyObject *value1 = NULL, *value2 = NULL; #if CYTHON_COMPILING_IN_PYPY - value1 = PySequence_ITEM(tuple, 0); - if (unlikely(!value1)) goto bad; - value2 = PySequence_ITEM(tuple, 1); - if (unlikely(!value2)) goto bad; -#else - value1 = PyTuple_GET_ITEM(tuple, 0); - value2 = PyTuple_GET_ITEM(tuple, 1); - Py_INCREF(value1); - Py_INCREF(value2); + value1 = PySequence_ITEM(tuple, 0); if (unlikely(!value1)) goto bad; + value2 = PySequence_ITEM(tuple, 1); if (unlikely(!value2)) goto bad; +#else + value1 = PyTuple_GET_ITEM(tuple, 0); Py_INCREF(value1); + value2 = PyTuple_GET_ITEM(tuple, 1); Py_INCREF(value2); #endif - if (decref_tuple) { Py_DECREF(tuple); } + if (decref_tuple) { + Py_DECREF(tuple); } + + *pvalue1 = value1; + *pvalue2 = value2; + return 0; +#if CYTHON_COMPILING_IN_PYPY +bad: + Py_XDECREF(value1); + Py_XDECREF(value2); + if (decref_tuple) { Py_XDECREF(tuple); } + return -1; +#endif +} + +static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyObject** pvalue2, + int has_known_size, int decref_tuple) { + Py_ssize_t index; + PyObject *value1 = NULL, *value2 = NULL, *iter = NULL; + iternextfunc iternext; + + iter = PyObject_GetIter(tuple); + if (unlikely(!iter)) goto bad; + if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } + + iternext = __Pyx_PyObject_GetIterNextFunc(iter); + value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } + value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } + if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; + + Py_DECREF(iter); *pvalue1 = value1; *pvalue2 = value2; return 0; + unpacking_failed: if (!has_known_size && __Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index); @@ -134,49 +153,77 @@ return -1; } + /////////////// IterNext.proto /////////////// #define __Pyx_PyIter_Next(obj) __Pyx_PyIter_Next2(obj, NULL) static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject *, PyObject *); /*proto*/ /////////////// IterNext /////////////// +//@requires: Exceptions.c::PyThreadStateGet +//@requires: Exceptions.c::PyErrFetchRestore + +static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) { + PyObject* exc_type; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + exc_type = __Pyx_PyErr_Occurred(); + if (unlikely(exc_type)) { + if (!defval || unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) + return NULL; + __Pyx_PyErr_Clear(); + Py_INCREF(defval); + return defval; + } + if (defval) { + Py_INCREF(defval); + return defval; + } + __Pyx_PyErr_SetNone(PyExc_StopIteration); + return NULL; +} + +static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) { + __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetName(Py_TYPE(iterator)); + PyErr_Format(PyExc_TypeError, + __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name); + __Pyx_DECREF_TypeName(iterator_type_name); +} // originally copied from Py3's builtin_next() static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* defval) { PyObject* next; + // We always do a quick slot check because calling PyIter_Check() is so wasteful. iternextfunc iternext = Py_TYPE(iterator)->tp_iternext; + if (likely(iternext)) { #if CYTHON_USE_TYPE_SLOTS - if (unlikely(!iternext)) { + next = iternext(iterator); + if (likely(next)) + return next; + if (unlikely(iternext == &_PyObject_NextNotImplemented)) + return NULL; #else - if (unlikely(!iternext) || unlikely(!PyIter_Check(iterator))) { -#endif - PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", Py_TYPE(iterator)->tp_name); + // Since the slot was set, assume that PyIter_Next() will likely succeed, and properly fail otherwise. + // Note: PyIter_Next() crashes in CPython if "tp_iternext" is NULL. + next = PyIter_Next(iterator); + if (likely(next)) + return next; +#endif + } else if (CYTHON_USE_TYPE_SLOTS || unlikely(!PyIter_Check(iterator))) { + // If CYTHON_USE_TYPE_SLOTS, then the slot was not set and we don't have an iterable. + // Otherwise, don't trust "tp_iternext" and rely on PyIter_Check(). + __Pyx_PyIter_Next_ErrorNoIterator(iterator); return NULL; } - next = iternext(iterator); - if (likely(next)) - return next; -#if CYTHON_USE_TYPE_SLOTS -#if PY_VERSION_HEX >= 0x02070000 - if (unlikely(iternext == &_PyObject_NextNotImplemented)) - return NULL; -#endif -#endif - if (defval) { - PyObject* exc_type = PyErr_Occurred(); - if (exc_type) { - if (unlikely(exc_type != PyExc_StopIteration) && - !PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) - return NULL; - PyErr_Clear(); - } - Py_INCREF(defval); - return defval; +#if !CYTHON_USE_TYPE_SLOTS + else { + // We have an iterator with an empty "tp_iternext", but didn't call next() on it yet. + next = PyIter_Next(iterator); + if (likely(next)) + return next; } - if (!PyErr_Occurred()) - PyErr_SetNone(PyExc_StopIteration); - return NULL; +#endif + return __Pyx_PyIter_Next2Default(defval); } /////////////// IterFinish.proto /////////////// @@ -191,10 +238,10 @@ static CYTHON_INLINE int __Pyx_IterFinish(void) { #if CYTHON_FAST_THREAD_STATE - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = __Pyx_PyThreadState_Current; PyObject* exc_type = tstate->curexc_type; if (unlikely(exc_type)) { - if (likely(exc_type == PyExc_StopIteration) || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) { + if (likely(__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) { PyObject *exc_value, *exc_tb; exc_value = tstate->curexc_value; exc_tb = tstate->curexc_traceback; @@ -223,29 +270,119 @@ #endif } + +/////////////// ObjectGetItem.proto /////////////// + +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key);/*proto*/ +#else +#define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) +#endif + +/////////////// ObjectGetItem /////////////// +// //@requires: GetItemInt - added in IndexNode as it uses templating. +//@requires: PyObjectGetAttrStrNoError +//@requires: PyObjectCallOneArg + +#if CYTHON_USE_TYPE_SLOTS +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + // Get element from sequence object `obj` at index `index`. + PyObject *runerr; + Py_ssize_t key_value; + key_value = __Pyx_PyIndex_AsSsize_t(index); + if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { + return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1); + } + + // Error handling code -- only manage OverflowError differently. + if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetName(Py_TYPE(index)); + PyErr_Clear(); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} + +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + // Handles less common slow-path checks for GetItem + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, PYIDENT("__class_getitem__")); + if (meth) { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } + } + + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} + +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); +} +#endif + + /////////////// DictGetItem.proto /////////////// #if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key);/*proto*/ + +#define __Pyx_PyObject_Dict_GetItem(obj, name) \ + (likely(PyDict_CheckExact(obj)) ? \ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) + +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/////////////// DictGetItem /////////////// + +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { PyObject *value; value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (!PyErr_Occurred()) { - PyObject* args = PyTuple_Pack(1, key); - if (likely(args)) - PyErr_SetObject(PyExc_KeyError, args); - Py_XDECREF(args); + if (unlikely(PyTuple_Check(key))) { + // CPython interprets tuples as separate arguments => must wrap them in another tuple. + PyObject* args = PyTuple_Pack(1, key); + if (likely(args)) { + PyErr_SetObject(PyExc_KeyError, args); + Py_DECREF(args); + } + } else { + // Avoid tuple packing if possible. + PyErr_SetObject(PyExc_KeyError, key); + } } return NULL; } Py_INCREF(value); return value; } -#else - #define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) #endif /////////////// GetItemInt.proto /////////////// +//@substitute: tempita #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck) \ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ? \ @@ -263,15 +400,16 @@ int wraparound, int boundscheck); {{endfor}} -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); /////////////// GetItemInt /////////////// +//@substitute: tempita -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; - if (!j) return NULL; + if (unlikely(!j)) return NULL; r = PyObject_GetItem(o, j); Py_DECREF(j); return r; @@ -282,9 +420,12 @@ CYTHON_NCP_UNUSED int wraparound, CYTHON_NCP_UNUSED int boundscheck) { #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - if (wraparound & unlikely(i < 0)) i += Py{{type}}_GET_SIZE(o); - if ((!boundscheck) || likely((0 <= i) & (i < Py{{type}}_GET_SIZE(o)))) { - PyObject *r = Py{{type}}_GET_ITEM(o, i); + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += Py{{type}}_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, Py{{type}}_GET_SIZE(o)))) { + PyObject *r = Py{{type}}_GET_ITEM(o, wrapped_i); Py_INCREF(r); return r; } @@ -301,7 +442,7 @@ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS if (is_list || PyList_CheckExact(o)) { Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); - if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { + if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { PyObject *r = PyList_GET_ITEM(o, n); Py_INCREF(r); return r; @@ -309,17 +450,25 @@ } else if (PyTuple_CheckExact(o)) { Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); - if ((!boundscheck) || likely((n >= 0) & (n < PyTuple_GET_SIZE(o)))) { + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { PyObject *r = PyTuple_GET_ITEM(o, n); Py_INCREF(r); return r; } } else { // inlined PySequence_GetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_subscript) { + PyObject *r, *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -329,7 +478,7 @@ PyErr_Clear(); } } - return m->sq_item(o, i); + return sm->sq_item(o, i); } } #else @@ -348,15 +497,15 @@ (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) : \ __Pyx_SetItemInt_Generic(o, to_py_func(i), v))) -static CYTHON_INLINE int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v); static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObject *v, int is_list, int wraparound, int boundscheck); /////////////// SetItemInt /////////////// -static CYTHON_INLINE int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { +static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_SetItem(o, j, v); Py_DECREF(j); return r; @@ -367,7 +516,7 @@ #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS if (is_list || PyList_CheckExact(o)) { Py_ssize_t n = (!wraparound) ? i : ((likely(i >= 0)) ? i : i + PyList_GET_SIZE(o)); - if ((!boundscheck) || likely((n >= 0) & (n < PyList_GET_SIZE(o)))) { + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o)))) { PyObject* old = PyList_GET_ITEM(o, n); Py_INCREF(v); PyList_SET_ITEM(o, n, v); @@ -376,10 +525,19 @@ } } else { // inlined PySequence_SetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -389,15 +547,16 @@ PyErr_Clear(); } } - return m->sq_ass_item(o, i, v); + return sm->sq_ass_item(o, i, v); } } #else #if CYTHON_COMPILING_IN_PYPY - if (is_list || (PySequence_Check(o) && !PyDict_Check(o))) { + if (is_list || (PySequence_Check(o) && !PyDict_Check(o))) #else - if (is_list || PySequence_Check(o)) { + if (is_list || PySequence_Check(o)) #endif + { return PySequence_SetItem(o, i, v); } #endif @@ -413,32 +572,37 @@ (is_list ? (PyErr_SetString(PyExc_IndexError, "list assignment index out of range"), -1) : \ __Pyx_DelItem_Generic(o, to_py_func(i)))) -static CYTHON_INLINE int __Pyx_DelItem_Generic(PyObject *o, PyObject *j); +static int __Pyx_DelItem_Generic(PyObject *o, PyObject *j); static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound); /////////////// DelItemInt /////////////// -static CYTHON_INLINE int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) { +static int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_DelItem(o, j); Py_DECREF(j); return r; } static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, - CYTHON_UNUSED int is_list, CYTHON_NCP_UNUSED int wraparound) { + int is_list, CYTHON_NCP_UNUSED int wraparound) { #if !CYTHON_USE_TYPE_SLOTS if (is_list || PySequence_Check(o)) { return PySequence_DelItem(o, i); } #else // inlined PySequence_DelItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if ((!is_list) && mm && mm->mp_ass_subscript) { + PyObject *key = PyInt_FromSsize_t(i); + return likely(key) ? mm->mp_ass_subscript(o, key, (PyObject *)NULL) : -1; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -448,7 +612,7 @@ PyErr_Clear(); } } - return m->sq_ass_item(o, i, (PyObject *)NULL); + return sm->sq_ass_item(o, i, (PyObject *)NULL); } #endif return __Pyx_DelItem_Generic(o, PyInt_FromSsize_t(i)); @@ -483,7 +647,8 @@ {{endif}} Py_ssize_t cstart, Py_ssize_t cstop, PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, - int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + int has_cstart, int has_cstop, int wraparound) { + __Pyx_TypeName obj_type_name; #if CYTHON_USE_TYPE_SLOTS PyMappingMethods* mp; #if PY_MAJOR_VERSION < 3 @@ -527,6 +692,8 @@ return ms->sq_ass_slice(obj, cstart, cstop, value); {{endif}} } +#else + CYTHON_UNUSED_VAR(wraparound); #endif mp = Py_TYPE(obj)->tp_as_mapping; @@ -535,6 +702,8 @@ {{else}} if (likely(mp && mp->mp_ass_subscript)) {{endif}} +#else + CYTHON_UNUSED_VAR(wraparound); #endif { {{if access == 'Get'}}PyObject*{{else}}int{{endif}} result; @@ -586,19 +755,70 @@ } return result; } + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, {{if access == 'Get'}} - "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); {{else}} - "'%.200s' object does not support slice %.10s", - Py_TYPE(obj)->tp_name, value ? "assignment" : "deletion"); + "'" __Pyx_FMT_TYPENAME "' object does not support slice %.10s", + obj_type_name, value ? "assignment" : "deletion"); {{endif}} + __Pyx_DECREF_TypeName(obj_type_name); bad: return {{if access == 'Get'}}NULL{{else}}-1{{endif}}; } +/////////////// TupleAndListFromArray.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/////////////// TupleAndListFromArray /////////////// +//@substitute: naming + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} + +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + Py_INCREF($empty_tuple); + return $empty_tuple; + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} + +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + + /////////////// SliceTupleAndList.proto /////////////// #if CYTHON_COMPILING_IN_CPYTHON @@ -610,6 +830,8 @@ #endif /////////////// SliceTupleAndList /////////////// +//@requires: TupleAndListFromArray +//@substitute: tempita #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) { @@ -630,32 +852,12 @@ *_stop = stop; } -static CYTHON_INLINE void __Pyx_copy_object_array(PyObject** CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { - PyObject *v; - Py_ssize_t i; - for (i = 0; i < length; i++) { - v = dest[i] = src[i]; - Py_INCREF(v); - } -} - {{for type in ['List', 'Tuple']}} static CYTHON_INLINE PyObject* __Pyx_Py{{type}}_GetSlice( PyObject* src, Py_ssize_t start, Py_ssize_t stop) { - PyObject* dest; Py_ssize_t length = Py{{type}}_GET_SIZE(src); __Pyx_crop_slice(&start, &stop, &length); - if (unlikely(length <= 0)) - return Py{{type}}_New(0); - - dest = Py{{type}}_New(length); - if (unlikely(!dest)) - return NULL; - __Pyx_copy_object_array( - ((Py{{type}}Object*)src)->ob_item + start, - ((Py{{type}}Object*)dest)->ob_item, - length); - return dest; + return __Pyx_Py{{type}}_FromArray(((Py{{type}}Object*)src)->ob_item + start, length); } {{endfor}} #endif @@ -766,7 +968,7 @@ //@requires: CalculateMetaclass static PyObject *__Pyx_Py3MetaclassGet(PyObject *bases, PyObject *mkw) { - PyObject *metaclass = mkw ? PyDict_GetItem(mkw, PYIDENT("metaclass")) : NULL; + PyObject *metaclass = mkw ? __Pyx_PyDict_GetItemStr(mkw, PYIDENT("metaclass")) : NULL; if (metaclass) { Py_INCREF(metaclass); if (PyDict_DelItem(mkw, PYIDENT("metaclass")) < 0) { @@ -797,13 +999,17 @@ PyObject *result; PyObject *metaclass; - if (PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0) + if (unlikely(PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0)) return NULL; - if (PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0) +#if PY_VERSION_HEX >= 0x03030000 + if (unlikely(PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0)) return NULL; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif /* Python2 __metaclass__ */ - metaclass = PyDict_GetItem(dict, PYIDENT("__metaclass__")); + metaclass = __Pyx_PyDict_GetItemStr(dict, PYIDENT("__metaclass__")); if (metaclass) { Py_INCREF(metaclass); if (PyType_Check(metaclass)) { @@ -821,6 +1027,94 @@ return result; } +/////////////// Py3UpdateBases.proto /////////////// + +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); /* proto */ + +/////////////// Py3UpdateBases ///////////////////// +//@requires: PyObjectCallOneArg +//@requires: PyObjectGetAttrStrNoError + +/* Shamelessly adapted from cpython/bltinmodule.c update_bases */ +static PyObject* +__Pyx_PEP560_update_bases(PyObject *bases) +{ + Py_ssize_t i, j, size_bases; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; + /*assert(PyTuple_Check(bases));*/ + + size_bases = PyTuple_GET_SIZE(bases); + for (i = 0; i < size_bases; i++) { + // original code in CPython: base = args[i]; + base = PyTuple_GET_ITEM(bases, i); + if (PyType_Check(base)) { + if (new_bases) { + // If we already have made a replacement, then we append every normal base, + // otherwise just skip it. + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + // original code in CPython: + // if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + meth = __Pyx_PyObject_GetAttrStrNoError(base, PYIDENT("__mro_entries__")); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + // If this is a first successful replacement, create new_bases list and + // copy previously encountered bases. + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + // original code in CPython: base = args[j]; + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + // unlike the CPython implementation, always return a new reference + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; + +error: + Py_XDECREF(new_bases); + return NULL; +} + /////////////// Py3ClassCreate.proto /////////////// static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, @@ -829,27 +1123,27 @@ PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); /*proto*/ /////////////// Py3ClassCreate /////////////// -//@requires: PyObjectGetAttrStr +//@substitute: naming +//@requires: PyObjectGetAttrStrNoError //@requires: CalculateMetaclass +//@requires: PyObjectFastCall +//@requires: PyObjectCall2Args +//@requires: PyObjectLookupSpecial +// only in fallback code: +//@requires: GetBuiltinName static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { PyObject *ns; if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, PYIDENT("__prepare__")); + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, PYIDENT("__prepare__")); if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); Py_DECREF(prep); - Py_DECREF(pargs); } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + if (unlikely(PyErr_Occurred())) return NULL; - PyErr_Clear(); ns = PyDict_New(); } } else { @@ -861,7 +1155,11 @@ /* Required here to emulate assignment order */ if (unlikely(PyObject_SetItem(ns, PYIDENT("__module__"), modname) < 0)) goto bad; +#if PY_VERSION_HEX >= 0x03030000 if (unlikely(PyObject_SetItem(ns, PYIDENT("__qualname__"), qualname) < 0)) goto bad; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif if (unlikely(doc && PyObject_SetItem(ns, PYIDENT("__doc__"), doc) < 0)) goto bad; return ns; bad: @@ -869,11 +1167,164 @@ return NULL; } +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +// https://www.python.org/dev/peps/pep-0487/ +static int __Pyx_SetNamesPEP487(PyObject *type_obj) { + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *names_to_set, *key, *value, *set_name, *tmp; + Py_ssize_t i = 0; + +#if CYTHON_USE_TYPE_SLOTS + names_to_set = PyDict_Copy(type->tp_dict); +#else + { + PyObject *d = PyObject_GetAttr(type_obj, PYIDENT("__dict__")); + names_to_set = NULL; + if (likely(d)) { + // d may not be a dict, e.g. PyDictProxy in PyPy2. + PyObject *names_to_set = PyDict_New(); + int ret = likely(names_to_set) ? PyDict_Update(names_to_set, d) : -1; + Py_DECREF(d); + if (unlikely(ret < 0)) + Py_CLEAR(names_to_set); + } + } +#endif + if (unlikely(names_to_set == NULL)) + goto bad; + + while (PyDict_Next(names_to_set, &i, &key, &value)) { + set_name = __Pyx_PyObject_LookupSpecialNoError(value, PYIDENT("__set_name__")); + if (unlikely(set_name != NULL)) { + tmp = __Pyx_PyObject_Call2Args(set_name, type_obj, key); + Py_DECREF(set_name); + if (unlikely(tmp == NULL)) { + __Pyx_TypeName value_type_name = + __Pyx_PyType_GetName(Py_TYPE(value)); + __Pyx_TypeName type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_RuntimeError, +#if PY_MAJOR_VERSION >= 3 + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %R " "in '" __Pyx_FMT_TYPENAME "'", + value_type_name, key, type_name); +#else + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %.100s in '" __Pyx_FMT_TYPENAME "'", + value_type_name, + PyString_Check(key) ? PyString_AS_STRING(key) : "?", + type_name); +#endif + goto bad; + } else { + Py_DECREF(tmp); + } + } + else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + + Py_DECREF(names_to_set); + return 0; +bad: + Py_XDECREF(names_to_set); + return -1; +} + +static PyObject *__Pyx_InitSubclassPEP487(PyObject *type_obj, PyObject *mkw) { +#if CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +// Stripped-down version of "super(type_obj, type_obj).__init_subclass__(**mkw)" in CPython 3.8. + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *mro = type->tp_mro; + Py_ssize_t i, nbases; + if (unlikely(!mro)) goto done; + + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; + + Py_INCREF(mro); + nbases = PyTuple_GET_SIZE(mro); + + // Skip over the type itself and 'object'. + assert(PyTuple_GET_ITEM(mro, 0) == type_obj); + for (i = 1; i < nbases-1; i++) { + PyObject *base, *dict, *meth; + base = PyTuple_GET_ITEM(mro, i); + dict = ((PyTypeObject *)base)->tp_dict; + meth = __Pyx_PyDict_GetItemStrWithError(dict, PYIDENT("__init_subclass__")); + if (unlikely(meth)) { + descrgetfunc f = Py_TYPE(meth)->tp_descr_get; + PyObject *res; + Py_INCREF(meth); + if (likely(f)) { + res = f(meth, NULL, type_obj); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + meth = res; + } + res = __Pyx_PyObject_FastCallDict(meth, NULL, 0, mkw); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + Py_DECREF(res); + goto done; + } else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + +done: + Py_XDECREF(mro); + return type_obj; + +bad: + Py_XDECREF(mro); + Py_DECREF(type_obj); + return NULL; + +// CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +#else +// Generic fallback: "super(type_obj, type_obj).__init_subclass__(**mkw)", as used in CPython 3.8. + PyObject *super_type, *super, *func, *res; + +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + super_type = __Pyx_GetBuiltinName(PYIDENT("super")); +#else + super_type = (PyObject*) &PySuper_Type; + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif + super = likely(super_type) ? __Pyx_PyObject_Call2Args(super_type, type_obj, type_obj) : NULL; +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + Py_XDECREF(super_type); +#endif + if (unlikely(!super)) { + Py_CLEAR(type_obj); + goto done; + } + func = __Pyx_PyObject_GetAttrStrNoError(super, PYIDENT("__init_subclass__")); + Py_DECREF(super); + if (likely(!func)) { + if (unlikely(PyErr_Occurred())) + Py_CLEAR(type_obj); + goto done; + } + res = __Pyx_PyObject_FastCallDict(func, NULL, 0, mkw); + Py_DECREF(func); + if (unlikely(!res)) + Py_CLEAR(type_obj); + Py_XDECREF(res); +done: + return type_obj; +#endif +} + +// PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +#endif + static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; + PyObject *result; PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; if (allow_py2_metaclass) { /* honour Python2 __metaclass__ for backward compatibility */ owned_metaclass = PyObject_GetItem(dict, PYIDENT("__metaclass__")); @@ -892,14 +1343,28 @@ return NULL; owned_metaclass = metaclass; } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; - } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); - } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, +#if PY_VERSION_HEX < 0x030600A4 + // Before PEP-487, type(a,b,c) did not accept any keyword arguments, so guard at least against that case. + (metaclass == (PyObject*)&PyType_Type) ? NULL : mkw +#else + mkw +#endif + ); Py_XDECREF(owned_metaclass); + +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS + if (likely(result) && likely(PyType_Check(result))) { + if (unlikely(__Pyx_SetNamesPEP487(result) < 0)) { + Py_CLEAR(result); + } else { + result = __Pyx_InitSubclassPEP487(result, mkw); + } + } +#else + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif return result; } @@ -910,21 +1375,28 @@ /////////////// ExtTypeTest /////////////// static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } - if (likely(PyObject_TypeCheck(obj, type))) + if (likely(__Pyx_TypeCheck(obj, type))) return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); return 0; } /////////////// CallableCheck.proto /////////////// #if CYTHON_USE_TYPE_SLOTS && PY_MAJOR_VERSION >= 3 -#define __Pyx_PyCallable_Check(obj) ((obj)->ob_type->tp_call != NULL) +#define __Pyx_PyCallable_Check(obj) (Py_TYPE(obj)->tp_call != NULL) #else #define __Pyx_PyCallable_Check(obj) PyCallable_Check(obj) #endif @@ -936,6 +1408,37 @@ return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); } +/////////////// PySetContains.proto /////////////// + +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq); /* proto */ + +/////////////// PySetContains /////////////// +//@requires: Builtins.c::pyfrozenset_new + +static int __Pyx_PySet_ContainsUnhashable(PyObject *set, PyObject *key) { + int result = -1; + if (PySet_Check(key) && PyErr_ExceptionMatches(PyExc_TypeError)) { + /* Convert key to frozenset */ + PyObject *tmpkey; + PyErr_Clear(); + tmpkey = __Pyx_PyFrozenSet_New(key); + if (tmpkey != NULL) { + result = PySet_Contains(set, tmpkey); + Py_DECREF(tmpkey); + } + } + return result; +} + +static CYTHON_INLINE int __Pyx_PySet_ContainsTF(PyObject* key, PyObject* set, int eq) { + int result = PySet_Contains(set, key); + + if (unlikely(result < 0)) { + result = __Pyx_PySet_ContainsUnhashable(set, key); + } + return unlikely(result < 0) ? result : (result == (eq == Py_EQ)); +} + /////////////// PySequenceContains.proto /////////////// static CYTHON_INLINE int __Pyx_PySequence_ContainsTF(PyObject* item, PyObject* seq, int eq) { @@ -954,12 +1457,12 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ /////////////// GetBuiltinName /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@substitute: naming static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr($builtins_cname, name); - if (unlikely(!result)) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError($builtins_cname, name); + if (unlikely(!result) && !PyErr_Occurred()) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); @@ -972,43 +1475,144 @@ /////////////// GetNameInClass.proto /////////////// -static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ +#define __Pyx_GetNameInClass(var, nmspace, name) (var) = __Pyx__GetNameInClass(nmspace, name) +static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ /////////////// GetNameInClass /////////////// -//@requires: PyObjectGetAttrStr //@requires: GetModuleGlobalName -static PyObject *__Pyx_GetNameInClass(PyObject *nmspace, PyObject *name) { +static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) - result = __Pyx_GetModuleGlobalName(name); - return result; + PyObject *dict; + assert(PyType_Check(nmspace)); +#if CYTHON_USE_TYPE_SLOTS + dict = ((PyTypeObject*)nmspace)->tp_dict; + Py_XINCREF(dict); +#else + dict = PyObject_GetAttr(nmspace, PYIDENT("__dict__")); +#endif + if (likely(dict)) { + result = PyObject_GetItem(dict, name); + Py_DECREF(dict); + if (result) { + return result; + } + } + PyErr_Clear(); + __Pyx_GetModuleGlobalNameUncached(result, name); + return result; +} + + +/////////////// SetNameInClass.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 +// Identifier names are always interned and have a pre-calculated hash value. +#define __Pyx_SetNameInClass(ns, name, value) \ + (likely(PyDict_CheckExact(ns)) ? _PyDict_SetItem_KnownHash(ns, name, value, ((PyASCIIObject *) name)->hash) : PyObject_SetItem(ns, name, value)) +#elif CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_SetNameInClass(ns, name, value) \ + (likely(PyDict_CheckExact(ns)) ? PyDict_SetItem(ns, name, value) : PyObject_SetItem(ns, name, value)) +#else +#define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) +#endif + +/////////////// SetNewInClass.proto /////////////// + +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value); + +/////////////// SetNewInClass /////////////// +//@requires: SetNameInClass + +// Special-case setting __new__: if it's a Cython function, wrap it in a +// staticmethod. This is similar to what Python does for a Python function +// called __new__. +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value) { +#ifdef __Pyx_CyFunction_USED + int ret; + if (__Pyx_CyFunction_Check(value)) { + PyObject *staticnew = PyStaticMethod_New(value); + if (unlikely(!staticnew)) return -1; + ret = __Pyx_SetNameInClass(ns, name, staticnew); + Py_DECREF(staticnew); + return ret; + } +#endif + return __Pyx_SetNameInClass(ns, name, value); } + /////////////// GetModuleGlobalName.proto /////////////// +//@requires: PyDictVersioning +//@substitute: naming + +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) { \ + static PY_UINT64_T __pyx_dict_version = 0; \ + static PyObject *__pyx_dict_cached_value = NULL; \ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION($moddict_cname))) ? \ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) : \ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value); \ +} +#define __Pyx_GetModuleGlobalNameUncached(var, name) { \ + PY_UINT64_T __pyx_dict_version; \ + PyObject *__pyx_dict_cached_value; \ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value); \ +} +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); /*proto*/ +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); /*proto*/ +#endif -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); /*proto*/ /////////////// GetModuleGlobalName /////////////// //@requires: GetBuiltinName //@substitute: naming -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ PyObject *result; +// FIXME: clean up the macro guard order here: limited API first, then borrowed refs, then cpython #if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + // Identifier names are always interned and have a pre-calculated hash value. + result = _PyDict_GetItem_KnownHash($moddict_cname, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#elif CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!$module_cname)) { + return NULL; + } + result = PyObject_GetAttr($module_cname, name); + if (likely(result)) { + return result; + } +#else result = PyDict_GetItem($moddict_cname, name); + __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) if (likely(result)) { - Py_INCREF(result); - } else { + return __Pyx_NewRef(result); + } +#endif #else result = PyObject_GetItem($moddict_cname, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); + __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); } - return result; + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); } //////////////////// GetAttr.proto //////////////////// @@ -1019,7 +1623,7 @@ //@requires: PyObjectGetAttrStr static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_USE_TYPE_SLOTS #if PY_MAJOR_VERSION >= 3 if (likely(PyUnicode_Check(n))) #else @@ -1030,17 +1634,31 @@ return PyObject_GetAttr(o, n); } + /////////////// PyObjectLookupSpecial.proto /////////////// + +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) + +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); /*proto*/ + +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/////////////// PyObjectLookupSpecial /////////////// //@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 -// looks like calling _PyType_Lookup() isn't safe in Py<=2.6/3.1 -static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { PyObject *res; PyTypeObject *tp = Py_TYPE(obj); #if PY_MAJOR_VERSION < 3 if (unlikely(PyInstance_Check(obj))) - return __Pyx_PyObject_GetAttrStr(obj, attr_name); + return with_error ? __Pyx_PyObject_GetAttrStr(obj, attr_name) : __Pyx_PyObject_GetAttrStrNoError(obj, attr_name); #endif // adapted from CPython's special_lookup() in ceval.c res = _PyType_Lookup(tp, attr_name); @@ -1051,18 +1669,146 @@ } else { res = f(res, obj, (PyObject *)tp); } - } else { + } else if (with_error) { PyErr_SetObject(PyExc_AttributeError, attr_name); } return res; } +#endif + + +/////////////// PyObject_GenericGetAttrNoDict.proto /////////////// + +// Setting "tp_getattro" to anything but "PyObject_GenericGetAttr" disables fast method calls in Py3.7. +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); #else -#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +// No-args macro to allow function pointer assignment. +#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr #endif +/////////////// PyObject_GenericGetAttrNoDict /////////////// + +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 + +static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + __Pyx_TypeName type_name = __Pyx_PyType_GetName(tp); + PyErr_Format(PyExc_AttributeError, +#if PY_MAJOR_VERSION >= 3 + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, attr_name); +#else + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(attr_name)); +#endif + __Pyx_DECREF_TypeName(type_name); + return NULL; +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { + // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 3.6/3.7. + // To be used in the "tp_getattro" slot of extension types that have no instance dict and cannot be subclassed. + PyObject *descr; + PyTypeObject *tp = Py_TYPE(obj); + + if (unlikely(!PyString_Check(attr_name))) { + return PyObject_GenericGetAttr(obj, attr_name); + } + + assert(!tp->tp_dictoffset); + descr = _PyType_Lookup(tp, attr_name); + if (unlikely(!descr)) { + return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); + } + + Py_INCREF(descr); + + #if PY_MAJOR_VERSION < 3 + if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) + #endif + { + descrgetfunc f = Py_TYPE(descr)->tp_descr_get; + // Optimise for the non-descriptor case because it is faster. + if (unlikely(f)) { + PyObject *res = f(descr, obj, (PyObject *)tp); + Py_DECREF(descr); + return res; + } + } + return descr; +} +#endif + + +/////////////// PyObject_GenericGetAttr.proto /////////////// + +// Setting "tp_getattro" to anything but "PyObject_GenericGetAttr" disables fast method calls in Py3.7. +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name); +#else +// No-args macro to allow function pointer assignment. +#define __Pyx_PyObject_GenericGetAttr PyObject_GenericGetAttr +#endif + +/////////////// PyObject_GenericGetAttr /////////////// +//@requires: PyObject_GenericGetAttrNoDict + +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name) { + if (unlikely(Py_TYPE(obj)->tp_dictoffset)) { + return PyObject_GenericGetAttr(obj, attr_name); + } + return __Pyx_PyObject_GenericGetAttrNoDict(obj, attr_name); +} +#endif + + +/////////////// PyObjectGetAttrStrNoError.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name);/*proto*/ + +/////////////// PyObjectGetAttrStrNoError /////////////// +//@requires: PyObjectGetAttrStr +//@requires: Exceptions.c::PyThreadStateGet +//@requires: Exceptions.c::PyErrFetchRestore +//@requires: Exceptions.c::PyErrExceptionMatches + +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS && PY_VERSION_HEX >= 0x030700B1 + // _PyObject_GenericGetAttrWithDict() in CPython 3.7+ can avoid raising the AttributeError. + // See https://bugs.python.org/issue32544 + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +} + + /////////////// PyObjectGetAttrStr.proto /////////////// #if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name);/*proto*/ +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/////////////// PyObjectGetAttrStr /////////////// + +#if CYTHON_USE_TYPE_SLOTS static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { PyTypeObject* tp = Py_TYPE(obj); if (likely(tp->tp_getattro)) @@ -1073,14 +1819,22 @@ #endif return PyObject_GetAttr(obj, attr_name); } -#else -#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) #endif + /////////////// PyObjectSetAttrStr.proto /////////////// #if CYTHON_USE_TYPE_SLOTS -#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o,n,NULL) +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value);/*proto*/ +#else +#define __Pyx_PyObject_DelAttrStr(o,n) PyObject_DelAttr(o,n) +#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) +#endif + +/////////////// PyObjectSetAttrStr /////////////// + +#if CYTHON_USE_TYPE_SLOTS static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { PyTypeObject* tp = Py_TYPE(obj); if (likely(tp->tp_setattro)) @@ -1091,10 +1845,130 @@ #endif return PyObject_SetAttr(obj, attr_name, value); } +#endif + + +/////////////// PyObjectGetMethod.proto /////////////// + +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method);/*proto*/ + +/////////////// PyObjectGetMethod /////////////// +//@requires: PyObjectGetAttrStr + +static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { + PyObject *attr; +#if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; + // Copied from _PyObject_GetMethod() in CPython 3.7 + PyTypeObject *tp = Py_TYPE(obj); + PyObject *descr; + descrgetfunc f = NULL; + PyObject **dictptr, *dict; + int meth_found = 0; + + assert (*method == NULL); + + if (unlikely(tp->tp_getattro != PyObject_GenericGetAttr)) { + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; + } + if (unlikely(tp->tp_dict == NULL) && unlikely(PyType_Ready(tp) < 0)) { + return 0; + } + + descr = _PyType_Lookup(tp, name); + if (likely(descr != NULL)) { + Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#elif PY_MAJOR_VERSION >= 3 + // Repeating the condition below accommodates for MSVC's inability to test macros inside of macro expansions. + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) + #endif #else -#define __Pyx_PyObject_DelAttrStr(o,n) PyObject_DelAttr(o,n) -#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) + // "PyMethodDescr_Type" is not part of the C-API in Py2. + #ifdef __Pyx_CyFunction_USED + if (likely(PyFunction_Check(descr) || __Pyx_CyFunction_Check(descr))) + #else + if (likely(PyFunction_Check(descr))) + #endif +#endif + { + meth_found = 1; + } else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + } + } + + dictptr = _PyObject_GetDictPtr(obj); + if (dictptr != NULL && (dict = *dictptr) != NULL) { + Py_INCREF(dict); + attr = __Pyx_PyDict_GetItemStr(dict, name); + if (attr != NULL) { + Py_INCREF(attr); + Py_DECREF(dict); + Py_XDECREF(descr); + goto try_unpack; + } + Py_DECREF(dict); + } + + if (meth_found) { + *method = descr; + return 1; + } + + if (f != NULL) { + attr = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + goto try_unpack; + } + + if (likely(descr != NULL)) { + *method = descr; + return 0; + } + + type_name = __Pyx_PyType_GetName(tp); + PyErr_Format(PyExc_AttributeError, +#if PY_MAJOR_VERSION >= 3 + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); +#else + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(name)); +#endif + __Pyx_DECREF_TypeName(type_name); + return 0; + +// Generic fallback implementation using normal attribute lookup. +#else + attr = __Pyx_PyObject_GetAttrStr(obj, name); + goto try_unpack; +#endif + +try_unpack: +#if CYTHON_UNPACK_METHODS + // Even if we failed to avoid creating a bound method object, it's still worth unpacking it now, if possible. + if (likely(attr) && PyMethod_Check(attr) && likely(PyMethod_GET_SELF(attr) == obj)) { + PyObject *function = PyMethod_GET_FUNCTION(attr); + Py_INCREF(function); + Py_DECREF(attr); + *method = function; + return 1; + } #endif + *method = attr; + return 0; +} /////////////// UnpackUnboundCMethod.proto /////////////// @@ -1120,13 +1994,13 @@ target->method = method; #if CYTHON_COMPILING_IN_CPYTHON #if PY_MAJOR_VERSION >= 3 - // method dscriptor type isn't exported in Py2.x, cannot easily check the type there - if (likely(PyObject_TypeCheck(method, &PyMethodDescr_Type))) + // method descriptor type isn't exported in Py2.x, cannot easily check the type there + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) #endif { PyMethodDescrObject *descr = (PyMethodDescrObject*) method; target->func = descr->d_method->ml_meth; - target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); } #endif return 0; @@ -1138,13 +2012,19 @@ static PyObject* __Pyx__CallUnboundCMethod0(__Pyx_CachedCFunction* cfunc, PyObject* self); /*proto*/ #if CYTHON_COMPILING_IN_CPYTHON +// FASTCALL methods receive "&empty_tuple" as simple "PyObject[0]*" #define __Pyx_CallUnboundCMethod0(cfunc, self) \ - ((likely((cfunc)->func)) ? \ + (likely((cfunc)->func) ? \ (likely((cfunc)->flag == METH_NOARGS) ? (*((cfunc)->func))(self, NULL) : \ - (likely((cfunc)->flag == (METH_VARARGS | METH_KEYWORDS)) ? ((*(PyCFunctionWithKeywords)(cfunc)->func)(self, $empty_tuple, NULL)) : \ - ((cfunc)->flag == METH_VARARGS ? (*((cfunc)->func))(self, $empty_tuple) : \ - (PY_VERSION_HEX >= 0x030600B1 && (cfunc)->flag == METH_FASTCALL ? (*(__Pyx_PyCFunctionFast)(cfunc)->func)(self, &PyTuple_GET_ITEM($empty_tuple, 0), 0, NULL) : \ - __Pyx__CallUnboundCMethod0(cfunc, self))))) : \ + (PY_VERSION_HEX >= 0x030600B1 && likely((cfunc)->flag == METH_FASTCALL) ? \ + (PY_VERSION_HEX >= 0x030700A0 ? \ + (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)(cfunc)->func)(self, &$empty_tuple, 0) : \ + (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)(cfunc)->func)(self, &$empty_tuple, 0, NULL)) : \ + (PY_VERSION_HEX >= 0x030700A0 && (cfunc)->flag == (METH_FASTCALL | METH_KEYWORDS) ? \ + (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)(cfunc)->func)(self, &$empty_tuple, 0, NULL) : \ + (likely((cfunc)->flag == (METH_VARARGS | METH_KEYWORDS)) ? ((*(PyCFunctionWithKeywords)(void*)(PyCFunction)(cfunc)->func)(self, $empty_tuple, NULL)) : \ + ((cfunc)->flag == METH_VARARGS ? (*((cfunc)->func))(self, $empty_tuple) : \ + __Pyx__CallUnboundCMethod0(cfunc, self)))))) : \ __Pyx__CallUnboundCMethod0(cfunc, self)) #else #define __Pyx_CallUnboundCMethod0(cfunc, self) __Pyx__CallUnboundCMethod0(cfunc, self) @@ -1175,14 +2055,10 @@ /////////////// CallUnboundCMethod1.proto /////////////// -static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg); /*proto*/ +static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg);/*proto*/ #if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_CallUnboundCMethod1(cfunc, self, arg) \ - ((likely((cfunc)->func && (cfunc)->flag == METH_O)) ? (*((cfunc)->func))(self, arg) : \ - ((PY_VERSION_HEX >= 0x030600B1 && (cfunc)->func && (cfunc)->flag == METH_FASTCALL) ? \ - (*(__Pyx_PyCFunctionFast)(cfunc)->func)(self, &arg, 1, NULL) : \ - __Pyx__CallUnboundCMethod1(cfunc, self, arg))) +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg);/*proto*/ #else #define __Pyx_CallUnboundCMethod1(cfunc, self, arg) __Pyx__CallUnboundCMethod1(cfunc, self, arg) #endif @@ -1191,9 +2067,30 @@ //@requires: UnpackUnboundCMethod //@requires: PyObjectCall +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg) { + if (likely(cfunc->func)) { + int flag = cfunc->flag; + // Not using #ifdefs for PY_VERSION_HEX to avoid C compiler warnings about unused functions. + if (flag == METH_O) { + return (*(cfunc->func))(self, arg); + } else if ((PY_VERSION_HEX >= 0x030600B1) && flag == METH_FASTCALL) { + if ((PY_VERSION_HEX >= 0x030700A0)) { + return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, &arg, 1); + } else { + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); + } + } else if ((PY_VERSION_HEX >= 0x030700A0) && flag == (METH_FASTCALL | METH_KEYWORDS)) { + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); + } + } + return __Pyx__CallUnboundCMethod1(cfunc, self, arg); +} +#endif + static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg){ PyObject *args, *result = NULL; - if (unlikely(!cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; #if CYTHON_COMPILING_IN_CPYTHON if (cfunc->func && (cfunc->flag & METH_VARARGS)) { args = PyTuple_New(1); @@ -1201,7 +2098,7 @@ Py_INCREF(arg); PyTuple_SET_ITEM(args, 0, arg); if (cfunc->flag & METH_KEYWORDS) - result = (*(PyCFunctionWithKeywords)cfunc->func)(self, args, NULL); + result = (*(PyCFunctionWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, NULL); else result = (*cfunc->func)(self, args); } else { @@ -1224,168 +2121,228 @@ } -/////////////// PyObjectCallMethod0.proto /////////////// +/////////////// CallUnboundCMethod2.proto /////////////// -static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/ +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); /*proto*/ -/////////////// PyObjectCallMethod0 /////////////// -//@requires: PyObjectGetAttrStr -//@requires: PyObjectCallOneArg -//@requires: PyObjectCallNoArg +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030600B1 +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); /*proto*/ +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif -static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto bad; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *function = PyMethod_GET_FUNCTION(method); - result = __Pyx_PyObject_CallOneArg(function, self); - Py_DECREF(method); - return result; +/////////////// CallUnboundCMethod2 /////////////// +//@requires: UnpackUnboundCMethod +//@requires: PyObjectCall + +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030600B1 +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + if (likely(cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + #if PY_VERSION_HEX >= 0x030700A0 + return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, args, 2); + #else + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, 2, NULL); + #endif } + #if PY_VERSION_HEX >= 0x030700A0 + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, 2, NULL); + #endif } + return __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); +} +#endif + +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + PyObject *args, *result = NULL; + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + args = PyTuple_New(2); + if (unlikely(!args)) goto bad; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = (*(PyCFunctionWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, NULL); + else + result = (*cfunc->func)(self, args); + } else { + args = PyTuple_New(3); + if (unlikely(!args)) goto bad; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 1, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 2, arg2); + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); + } +#else + args = PyTuple_Pack(3, self, arg1, arg2); + if (unlikely(!args)) goto bad; + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); #endif - result = __Pyx_PyObject_CallNoArg(method); - Py_DECREF(method); bad: + Py_XDECREF(args); return result; } -/////////////// PyObjectCallMethod1.proto /////////////// +/////////////// PyObjectFastCall.proto /////////////// -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); /*proto*/ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs); /*proto*/ -/////////////// PyObjectCallMethod1 /////////////// -//@requires: PyObjectGetAttrStr -//@requires: PyObjectCallOneArg +/////////////// PyObjectFastCall /////////////// +//@requires: PyObjectCall //@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectCallMethO +//@substitute: naming -static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { - PyObject *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) goto done; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method))) { - PyObject *self = PyMethod_GET_SELF(method); - if (likely(self)) { - PyObject *args; - PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result; + size_t i; + + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]); + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + Py_DECREF(argstuple); + return result; +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t _nargs, PyObject *kwargs) { + // Special fast paths for 0 and 1 arguments + // NOTE: in many cases, this is called with a constant value for nargs + // which is known at compile-time. So the branches below will typically + // be optimized away. + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { +#ifdef __Pyx_CyFunction_USED + if (__Pyx_IsCyOrPyCFunction(func)) +#else + if (PyCFunction_Check(func)) +#endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; + } + } + else if (nargs == 1 && kwargs == NULL) { + if (PyCFunction_Check(func)) + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, args[0]); } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; } } #endif - result = __Pyx_PyObject_CallOneArg(method, arg); -done: - Py_XDECREF(method); - return result; + + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + #if PY_VERSION_HEX >= 0x030700A1 + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + + #if CYTHON_VECTORCALL + vectorcallfunc f = _PyVectorcall_Function(func); + if (f) { + return f(func, args, (size_t)nargs, kwargs); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + // exclude fused functions for now + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, (size_t)nargs, kwargs); + } + #endif + + if (nargs == 0) { + return __Pyx_PyObject_Call(func, $empty_tuple, kwargs); + } + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); } -/////////////// PyObjectCallMethod2.proto /////////////// +/////////////// PyObjectCallMethod0.proto /////////////// -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2); /*proto*/ +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/ -/////////////// PyObjectCallMethod2 /////////////// -//@requires: PyObjectGetAttrStr -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +/////////////// PyObjectCallMethod0 /////////////// +//@requires: PyObjectGetMethod +//@requires: PyObjectCallOneArg +//@requires: PyObjectCallNoArg -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2) { - PyObject *args, *method, *result = NULL; - method = __Pyx_PyObject_GetAttrStr(obj, method_name); - if (unlikely(!method)) return NULL; -#if CYTHON_UNPACK_METHODS - if (likely(PyMethod_Check(method)) && likely(PyMethod_GET_SELF(method))) { - PyObject *self, *function; - self = PyMethod_GET_SELF(method); - function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[3] = {self, arg1, arg2}; - result = __Pyx_PyFunction_FastCall(function, args, 3); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[3] = {self, arg1, arg2}; - result = __Pyx_PyFunction_FastCall(function, args, 3); - goto done; - } - #endif - args = PyTuple_New(3); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 1, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 2, arg2); - Py_INCREF(function); +static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name) { + PyObject *method = NULL, *result = NULL; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_CallOneArg(method, obj); Py_DECREF(method); - method = function; - } else -#endif -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(method)) { - PyObject *args[2] = {arg1, arg2}; - result = __Pyx_PyFunction_FastCall(method, args, 2); - goto done; - } else -#endif -#if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(method)) { - PyObject *args[2] = {arg1, arg2}; - result = __Pyx_PyCFunction_FastCall(method, args, 2); - goto done; - } else -#endif - { - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); + return result; } - result = __Pyx_PyObject_Call(method, args, NULL); - Py_DECREF(args); -done: + if (unlikely(!method)) goto bad; + result = __Pyx_PyObject_CallNoArg(method); + Py_DECREF(method); +bad: + return result; +} + + +/////////////// PyObjectCallMethod1.proto /////////////// + +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg); /*proto*/ + +/////////////// PyObjectCallMethod1 /////////////// +//@requires: PyObjectGetMethod +//@requires: PyObjectCallOneArg +//@requires: PyObjectCall2Args + +static PyObject* __Pyx__PyObject_CallMethod1(PyObject* method, PyObject* arg) { + // Separate function to avoid excessive inlining. + PyObject *result = __Pyx_PyObject_CallOneArg(method, arg); Py_DECREF(method); return result; } +static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name, PyObject* arg) { + PyObject *method = NULL, *result; + int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); + if (likely(is_method)) { + result = __Pyx_PyObject_Call2Args(method, obj, arg); + Py_DECREF(method); + return result; + } + if (unlikely(!method)) return NULL; + return __Pyx__PyObject_CallMethod1(method, arg); +} + /////////////// tp_new.proto /////////////// @@ -1408,7 +2365,7 @@ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { PyObject *result; - ternaryfunc call = func->ob_type->tp_call; + ternaryfunc call = Py_TYPE(func)->tp_call; if (unlikely(!call)) return PyObject_Call(func, arg, kw); @@ -1458,27 +2415,72 @@ /////////////// PyFunctionFastCall.proto /////////////// #if CYTHON_FAST_PYCALL + +#if !CYTHON_VECTORCALL #define __Pyx_PyFunction_FastCall(func, args, nargs) \ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -// let's assume that the non-public C-API function might still change during the 3.6 beta phase -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs); #endif + +// Backport from Python 3 +// Assert a build-time dependency, as an expression. +// Your compile will fail if the condition isn't true, or can't be evaluated +// by the compiler. This can be used in an expression: its value is 0. +// Example: +// #define foo_to_char(foo) \ +// ((char *)(foo) \ +// + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) +// +// Written by Rusty Russell, public domain, https://ccodearchive.net/ +#define __Pyx_BUILD_ASSERT_EXPR(cond) \ + (sizeof(char [1 - 2*!(cond)]) - 1) + +#ifndef Py_MEMBER_SIZE +// Get the size of a structure member in bytes +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif + +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" +#if PY_VERSION_HEX >= 0x030b00a6 + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + // Initialised by module init code. + static size_t __pyx_pyframe_localsplus_offset = 0; + + #include "frameobject.h" + // This is the long runtime version of + // #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) + // offsetof(PyFrameObject, f_localsplus) differs between regular C-Python and Stackless Python < 3.8. + // Therefore the offset is computed at run time from PyFrame_type.tp_basicsize. That is feasible, + // because f_localsplus is the last field of PyFrameObject (checked by Py_BUILD_ASSERT_EXPR below). + #define __Pxy_PyFrame_Initialize_Offsets() \ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)), \ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame) \ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) #endif +#endif /* !CYTHON_VECTORCALL */ + +#endif /* CYTHON_FAST_PYCALL */ + /////////////// PyFunctionFastCall /////////////// // copied from CPython 3.6 ceval.c -#if CYTHON_FAST_PYCALL -#include "frameobject.h" - +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, PyObject *globals) { PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); + PyThreadState *tstate = __Pyx_PyThreadState_Current; PyObject **fastlocals; Py_ssize_t i; PyObject *result; @@ -1494,7 +2496,7 @@ return NULL; } - fastlocals = f->f_localsplus; + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); for (i = 0; i < na; i++) { Py_INCREF(*args); @@ -1510,8 +2512,7 @@ } -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); @@ -1531,7 +2532,7 @@ assert(kwargs == NULL || PyDict_Check(kwargs)); nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) { return NULL; } @@ -1605,12 +2606,12 @@ //#elif PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3 result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, + args, (int)nargs, k, (int)nk, d, (int)nd, kwdefs, closure); #else result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, + args, (int)nargs, k, (int)nk, d, (int)nd, closure); #endif @@ -1620,41 +2621,20 @@ Py_LeaveRecursiveCall(); return result; } -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL - - -/////////////// PyCFunctionFastCall.proto /////////////// +#endif /* CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif -/////////////// PyCFunctionFastCall /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - PyObject *result; - int flags; +/////////////// PyObjectCall2Args.proto /////////////// - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); +/////////////// PyObjectCall2Args /////////////// +//@requires: PyObjectFastCall - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } -#endif // CYTHON_FAST_PYCCALL /////////////// PyObjectCallOneArg.proto /////////////// @@ -1662,91 +2642,97 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); /*proto*/ /////////////// PyObjectCallOneArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall - -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} +//@requires: PyObjectFastCall static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif /////////////// PyObjectCallNoArg.proto /////////////// -//@requires: PyObjectCall -//@substitute: naming -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); /*proto*/ -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, $empty_tuple, NULL) -#endif /////////////// PyObjectCallNoArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@substitute: naming +//@requires: PyObjectFastCall -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); + PyObject *arg = NULL; + return __Pyx_PyObject_FastCall(func, (&arg)+1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + + +/////////////// PyVectorcallFastCallDict.proto /////////////// + +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/////////////// PyVectorcallFastCallDict /////////////// + +#if CYTHON_METH_FASTCALL +// Slow path when kw is non-empty +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + // Code based on _PyObject_FastCallDict() and _PyStack_UnpackDict() from CPython + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + + // Copy positional arguments + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, NULL); - } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + + // Copy keyword arguments + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= Py_TYPE(key)->tp_flags; + Py_INCREF(key); + Py_INCREF(value); + PyTuple_SET_ITEM(kwnames, i, key); + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + + // The actual call + res = vc(func, newargs, nargs, kwnames); + +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} + +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + if (likely(kw == NULL) || PyDict_GET_SIZE(kw) == 0) { + return vc(func, args, nargs, NULL); } - return __Pyx_PyObject_Call(func, $empty_tuple, NULL); + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); } #endif @@ -1763,10 +2749,9 @@ #endif /////////////// MatrixMultiply /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@requires: PyObjectCallOneArg -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectCall2Args #if PY_VERSION_HEX < 0x03050000 static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg) { @@ -1776,34 +2761,9 @@ if (likely(PyMethod_Check(method))) { PyObject *self = PyMethod_GET_SELF(method); if (likely(self)) { - PyObject *args; PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; + result = __Pyx_PyObject_Call2Args(function, self, arg); + goto done; } } #endif @@ -1813,21 +2773,21 @@ return result; } -#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ - PyObject *func = __Pyx_PyObject_GetAttrStr(x, py_method_name); \ +#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ + PyObject *func = __Pyx_PyObject_GetAttrStrNoError(x, py_method_name); \ if (func) { \ PyObject *result = __Pyx_PyObject_CallMatrixMethod(func, y); \ if (result != Py_NotImplemented) \ return result; \ Py_DECREF(result); \ - } else { \ - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) \ - return NULL; \ - PyErr_Clear(); \ + } else if (unlikely(PyErr_Occurred())) { \ + return NULL; \ } \ } static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const char* op_name) { + __Pyx_TypeName x_type_name; + __Pyx_TypeName y_type_name; int right_is_subtype = PyObject_IsSubclass((PyObject*)Py_TYPE(y), (PyObject*)Py_TYPE(x)); if (unlikely(right_is_subtype == -1)) return NULL; @@ -1840,11 +2800,13 @@ if (!right_is_subtype) { __Pyx_TryMatrixMethod(y, x, PYIDENT("__rmatmul__")) } + x_type_name = __Pyx_PyType_GetName(Py_TYPE(x)); + y_type_name = __Pyx_PyType_GetName(Py_TYPE(y)); PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.2s: '%.100s' and '%.100s'", - op_name, - Py_TYPE(x)->tp_name, - Py_TYPE(y)->tp_name); + "unsupported operand type(s) for %.2s: '" __Pyx_FMT_TYPENAME "' and '" + __Pyx_FMT_TYPENAME "'", op_name, x_type_name, y_type_name); + __Pyx_DECREF_TypeName(x_type_name); + __Pyx_DECREF_TypeName(y_type_name); return NULL; } @@ -1855,3 +2817,298 @@ #undef __Pyx_TryMatrixMethod #endif + + +/////////////// PyDictVersioning.proto /////////////// + +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) \ + (version_var) = __PYX_GET_DICT_VERSION(dict); \ + (cache_var) = (value); + +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) { \ + static PY_UINT64_T __pyx_dict_version = 0; \ + static PyObject *__pyx_dict_cached_value = NULL; \ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) { \ + (VAR) = __pyx_dict_cached_value; \ + } else { \ + (VAR) = __pyx_dict_cached_value = (LOOKUP); \ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT); \ + } \ +} + +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); /*proto*/ +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); /*proto*/ +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); /*proto*/ + +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/////////////// PyDictVersioning /////////////// + +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} + +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} + +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + + +/////////////// PyMethodNew.proto /////////////// + +#if PY_MAJOR_VERSION >= 3 +// This should be an actual function (not a macro), such that we can put it +// directly in a tp_descr_get slot. +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#else + #define __Pyx_PyMethod_New PyMethod_New +#endif + +/////////////// UnicodeConcatInPlace.proto //////////////// + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// __Pyx_PyUnicode_ConcatInPlace may modify the first argument 'left' +// However, unlike `PyUnicode_Append` it will never NULL it. +// It behaves like a regular function - returns a new reference and NULL on error + #if CYTHON_REFNANNY + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, __pyx_refnanny) + #else + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right) + #endif + // __Pyx_PyUnicode_ConcatInPlace is slightly odd because it has the potential to modify the input + // argument (but only in cases where no user should notice). Therefore, it needs to keep Cython's + // refnanny informed. + static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ); /* proto */ +#else +#define __Pyx_PyUnicode_ConcatInPlace __Pyx_PyUnicode_Concat +#endif +#define __Pyx_PyUnicode_ConcatInPlaceSafe(left, right) ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ? \ + PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_ConcatInPlace(left, right)) + +/////////////// UnicodeConcatInPlace //////////////// +//@substitute: naming + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// copied directly from unicode_object.c "unicode_modifiable +// removing _PyUnicode_HASH since it's a macro we don't have +// - this is OK because trying PyUnicode_Resize on a non-modifyable +// object will still work, it just won't happen in place +static int +__Pyx_unicode_modifiable(PyObject *unicode) +{ + if (Py_REFCNT(unicode) != 1) + return 0; + if (!PyUnicode_CheckExact(unicode)) + return 0; + if (PyUnicode_CHECK_INTERNED(unicode)) + return 0; + return 1; +} + +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ) { + // heavily based on PyUnicode_Append + PyObject *left = *p_left; + Py_ssize_t left_len, right_len, new_len; + + if (unlikely(__Pyx_PyUnicode_READY(left) == -1)) + return NULL; + if (unlikely(__Pyx_PyUnicode_READY(right) == -1)) + return NULL; + + // Shortcuts + left_len = PyUnicode_GET_LENGTH(left); + if (left_len == 0) { + Py_INCREF(right); + return right; + } + right_len = PyUnicode_GET_LENGTH(right); + if (right_len == 0) { + Py_INCREF(left); + return left; + } + if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + return NULL; + } + new_len = left_len + right_len; + + if (__Pyx_unicode_modifiable(left) + && PyUnicode_CheckExact(right) + && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + // Don't resize for ascii += latin1. Convert ascii to latin1 requires + // to change the structure size, but characters are stored just after + // the structure, and so it requires to move all characters which is + // not so different than duplicating the string. + && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) { + + __Pyx_GIVEREF(*p_left); + if (unlikely(PyUnicode_Resize(p_left, new_len) != 0)) { + // on failure PyUnicode_Resize does not deallocate the the input + // so left will remain unchanged - simply undo the giveref + __Pyx_GOTREF(*p_left); + return NULL; + } + __Pyx_INCREF(*p_left); + + // copy 'right' into the newly allocated area of 'left' + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); + return *p_left; + } else { + return __Pyx_PyUnicode_Concat(left, right); + } + } +#endif + +////////////// StrConcatInPlace.proto /////////////////////// +//@requires: UnicodeConcatInPlace + +#if PY_MAJOR_VERSION >= 3 + // allow access to the more efficient versions where we know str_type is unicode + #define __Pyx_PyStr_Concat __Pyx_PyUnicode_Concat + #define __Pyx_PyStr_ConcatInPlace __Pyx_PyUnicode_ConcatInPlace +#else + #define __Pyx_PyStr_Concat PyNumber_Add + #define __Pyx_PyStr_ConcatInPlace PyNumber_InPlaceAdd +#endif +#define __Pyx_PyStr_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_Add(a, b) : __Pyx_PyStr_Concat(a, b)) +#define __Pyx_PyStr_ConcatInPlaceSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_InPlaceAdd(a, b) : __Pyx_PyStr_ConcatInPlace(a, b)) + +/////////////// FormatTypeName.proto /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +static __Pyx_TypeName __Pyx_PyType_GetName(PyTypeObject* tp); /*proto*/ +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#else +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/////////////// FormatTypeName /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +static __Pyx_TypeName +__Pyx_PyType_GetName(PyTypeObject* tp) +{ + PyObject *name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + PYIDENT("__name__")); + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) { + PyErr_Clear(); + Py_XSETREF(name, __Pyx_NewRef(PYIDENT("?"))); + } + return name; +} +#endif + + +/////////////// RaiseUnexpectedTypeError.proto /////////////// + +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); /*proto*/ + +/////////////// RaiseUnexpectedTypeError /////////////// + +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + + +/////////////// RaiseUnboundLocalError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);/*proto*/ + +/////////////// RaiseUnboundLocalError /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + + +/////////////// RaiseClosureNameError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);/*proto*/ + +/////////////// RaiseClosureNameError /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + + +/////////////// RaiseUnboundMemoryviewSliceNogil.proto /////////////// +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname);/*proto*/ + +/////////////// RaiseUnboundMemoryviewSliceNogil /////////////// +//@requires: RaiseUnboundLocalError + +// Don't inline the function, it should really never be called in production +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { + #ifdef WITH_THREAD + PyGILState_STATE gilstate = PyGILState_Ensure(); + #endif + __Pyx_RaiseUnboundLocalError(varname); + #ifdef WITH_THREAD + PyGILState_Release(gilstate); + #endif +} + +//////////////// RaiseCppGlobalNameError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname); /*proto*/ + +/////////////// RaiseCppGlobalNameError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "C++ global '%s' is not initialized", varname); +} + +//////////////// RaiseCppAttributeError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname); /*proto*/ + +/////////////// RaiseCppAttributeError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname) { + PyErr_Format(PyExc_AttributeError, "C++ attribute '%s' is not initialized", varname); +} diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Optimize.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Optimize.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Optimize.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Optimize.c 2022-03-24 10:16:46.000000000 +0000 @@ -35,7 +35,7 @@ if (likely(L->allocated > len) & likely(len > (L->allocated >> 1))) { Py_INCREF(x); PyList_SET_ITEM(list, len, x); - Py_SIZE(list) = len+1; + __Pyx_SET_SIZE(list, len + 1); return 0; } return PyList_Append(list, x); @@ -53,7 +53,7 @@ if (likely(L->allocated > len)) { Py_INCREF(x); PyList_SET_ITEM(list, len, x); - Py_SIZE(list) = len+1; + __Pyx_SET_SIZE(list, len + 1); return 0; } return PyList_Append(list, x); @@ -94,7 +94,7 @@ //@requires: ObjectHandling.c::PyObjectCallMethod0 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) { - if (Py_TYPE(L) == &PySet_Type) { + if (__Pyx_IS_TYPE(L, &PySet_Type)) { return PySet_Pop(L); } return __Pyx_PyObject_CallMethod0(L, PYIDENT("pop")); @@ -104,7 +104,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L) { /* Check that both the size is positive and no reallocation shrinking needs to be done. */ if (likely(PyList_GET_SIZE(L) > (((PyListObject*)L)->allocated >> 1))) { - Py_SIZE(L) -= 1; + __Pyx_SET_SIZE(L, Py_SIZE(L) - 1); return PyList_GET_ITEM(L, PyList_GET_SIZE(L)); } return CALL_UNBOUND_METHOD(PyList_Type, "pop", L); @@ -123,13 +123,13 @@ #define __Pyx_PyObject_PopIndex(L, py_ix, ix, is_signed, type, to_py_func) ( \ (likely(PyList_CheckExact(L) && __Pyx_fits_Py_ssize_t(ix, type, is_signed))) ? \ __Pyx__PyList_PopIndex(L, py_ix, ix) : ( \ - (unlikely(py_ix == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ + (unlikely((py_ix) == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ __Pyx__PyObject_PopIndex(L, py_ix))) #define __Pyx_PyList_PopIndex(L, py_ix, ix, is_signed, type, to_py_func) ( \ __Pyx_fits_Py_ssize_t(ix, type, is_signed) ? \ __Pyx__PyList_PopIndex(L, py_ix, ix) : ( \ - (unlikely(py_ix == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ + (unlikely((py_ix) == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ __Pyx__PyObject_PopIndex(L, py_ix))) #else @@ -138,7 +138,7 @@ __Pyx_PyObject_PopIndex(L, py_ix, ix, is_signed, type, to_py_func) #define __Pyx_PyObject_PopIndex(L, py_ix, ix, is_signed, type, to_py_func) ( \ - (unlikely(py_ix == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ + (unlikely((py_ix) == Py_None)) ? __Pyx__PyObject_PopNewIndex(L, to_py_func(ix)) : \ __Pyx__PyObject_PopIndex(L, py_ix)) #endif @@ -165,9 +165,9 @@ if (cix < 0) { cix += size; } - if (likely(0 <= cix && cix < size)) { + if (likely(__Pyx_is_valid_index(cix, size))) { PyObject* v = PyList_GET_ITEM(L, cix); - Py_SIZE(L) -= 1; + __Pyx_SET_SIZE(L, Py_SIZE(L) - 1); size -= 1; memmove(&PyList_GET_ITEM(L, cix), &PyList_GET_ITEM(L, cix+1), (size_t)(size-cix)*sizeof(PyObject*)); return v; @@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { PyObject* value; -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (unlikely(PyErr_Occurred())) @@ -198,6 +198,8 @@ value = default_value; } Py_INCREF(value); + // avoid C compiler warning about unused utility functions + if ((1)); #else if (PyString_CheckExact(key) || PyUnicode_CheckExact(key) || PyInt_CheckExact(key)) { /* these presumably have safe hash functions */ @@ -206,13 +208,14 @@ value = default_value; } Py_INCREF(value); - } else { - if (default_value == Py_None) - default_value = NULL; - value = PyObject_CallMethodObjArgs( - d, PYIDENT("get"), key, default_value, NULL); } #endif + else { + if (default_value == Py_None) + value = CALL_UNBOUND_METHOD(PyDict_Type, "get", d, key); + else + value = CALL_UNBOUND_METHOD(PyDict_Type, "get", d, key, default_value); + } return value; } @@ -222,21 +225,21 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); /*proto*/ /////////////// dict_setdefault /////////////// -//@requires: ObjectHandling.c::PyObjectCallMethod2 static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, - CYTHON_UNUSED int is_safe_type) { + int is_safe_type) { PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); #if PY_VERSION_HEX >= 0x030400A0 // we keep the method call at the end to avoid "unused" C compiler warnings - if (1) { + if ((1)) { value = PyDict_SetDefault(d, key, default_value); if (unlikely(!value)) return NULL; Py_INCREF(value); #else if (is_safe_type == 1 || (is_safe_type == -1 && /* the following builtins presumably have repeatably safe and fast hash functions */ -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) { value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { @@ -259,7 +262,7 @@ #endif #endif } else { - value = __Pyx_PyObject_CallMethod2(d, PYIDENT("setdefault"), key, default_value); + value = CALL_UNBOUND_METHOD(PyDict_Type, "setdefault", d, key, default_value); } return value; } @@ -269,6 +272,28 @@ #define __Pyx_PyDict_Clear(d) (PyDict_Clear(d), 0) + +/////////////// py_dict_pop.proto /////////////// + +static CYTHON_INLINE PyObject *__Pyx_PyDict_Pop(PyObject *d, PyObject *key, PyObject *default_value); /*proto*/ + +/////////////// py_dict_pop /////////////// + +static CYTHON_INLINE PyObject *__Pyx_PyDict_Pop(PyObject *d, PyObject *key, PyObject *default_value) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B3 + if ((1)) { + return _PyDict_Pop(d, key, default_value); + } else + // avoid "function unused" warnings +#endif + if (default_value) { + return CALL_UNBOUND_METHOD(PyDict_Type, "pop", d, key, default_value); + } else { + return CALL_UNBOUND_METHOD(PyDict_Type, "pop", d, key); + } +} + + /////////////// dict_iter.proto /////////////// static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* dict, int is_dict, PyObject* method_name, @@ -285,13 +310,32 @@ Py_ssize_t* p_orig_length, int* p_source_is_dict) { is_dict = is_dict || likely(PyDict_CheckExact(iterable)); *p_source_is_dict = is_dict; -#if !CYTHON_COMPILING_IN_PYPY if (is_dict) { +#if !CYTHON_COMPILING_IN_PYPY *p_orig_length = PyDict_Size(iterable); Py_INCREF(iterable); return iterable; - } +#elif PY_MAJOR_VERSION >= 3 + // On PyPy3, we need to translate manually a few method names. + // This logic is not needed on CPython thanks to the fast case above. + static PyObject *py_items = NULL, *py_keys = NULL, *py_values = NULL; + PyObject **pp = NULL; + if (method_name) { + const char *name = PyUnicode_AsUTF8(method_name); + if (strcmp(name, "iteritems") == 0) pp = &py_items; + else if (strcmp(name, "iterkeys") == 0) pp = &py_keys; + else if (strcmp(name, "itervalues") == 0) pp = &py_values; + if (pp) { + if (!*pp) { + *pp = PyUnicode_FromString(name + 4); + if (!*pp) + return NULL; + } + method_name = *pp; + } + } #endif + } *p_orig_length = 0; if (method_name) { PyObject* iter; @@ -378,6 +422,143 @@ } +/////////////// set_iter.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx_set_iterator(PyObject* iterable, int is_set, + Py_ssize_t* p_orig_length, int* p_source_is_set); /*proto*/ +static CYTHON_INLINE int __Pyx_set_iter_next( + PyObject* iter_obj, Py_ssize_t orig_length, + Py_ssize_t* ppos, PyObject **value, + int source_is_set); /*proto*/ + +/////////////// set_iter /////////////// +//@requires: ObjectHandling.c::IterFinish + +static CYTHON_INLINE PyObject* __Pyx_set_iterator(PyObject* iterable, int is_set, + Py_ssize_t* p_orig_length, int* p_source_is_set) { +#if CYTHON_COMPILING_IN_CPYTHON + is_set = is_set || likely(PySet_CheckExact(iterable) || PyFrozenSet_CheckExact(iterable)); + *p_source_is_set = is_set; + if (likely(is_set)) { + *p_orig_length = PySet_Size(iterable); + Py_INCREF(iterable); + return iterable; + } +#else + (void)is_set; + *p_source_is_set = 0; +#endif + *p_orig_length = 0; + return PyObject_GetIter(iterable); +} + +static CYTHON_INLINE int __Pyx_set_iter_next( + PyObject* iter_obj, Py_ssize_t orig_length, + Py_ssize_t* ppos, PyObject **value, + int source_is_set) { + if (!CYTHON_COMPILING_IN_CPYTHON || unlikely(!source_is_set)) { + *value = PyIter_Next(iter_obj); + if (unlikely(!*value)) { + return __Pyx_IterFinish(); + } + (void)orig_length; + (void)ppos; + return 1; + } +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(PySet_GET_SIZE(iter_obj) != orig_length)) { + PyErr_SetString( + PyExc_RuntimeError, + "set changed size during iteration"); + return -1; + } + { + Py_hash_t hash; + int ret = _PySet_NextEntry(iter_obj, ppos, value, &hash); + // CPython does not raise errors here, only if !isinstance(iter_obj, set/frozenset) + assert (ret != -1); + if (likely(ret)) { + Py_INCREF(*value); + return 1; + } + } +#endif + return 0; +} + +/////////////// py_set_discard_unhashable /////////////// +//@requires: Builtins.c::pyfrozenset_new + +static int __Pyx_PySet_DiscardUnhashable(PyObject *set, PyObject *key) { + PyObject *tmpkey; + int rv; + + if (likely(!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError))) + return -1; + PyErr_Clear(); + tmpkey = __Pyx_PyFrozenSet_New(key); + if (tmpkey == NULL) + return -1; + rv = PySet_Discard(set, tmpkey); + Py_DECREF(tmpkey); + return rv; +} + + +/////////////// py_set_discard.proto /////////////// + +static CYTHON_INLINE int __Pyx_PySet_Discard(PyObject *set, PyObject *key); /*proto*/ + +/////////////// py_set_discard /////////////// +//@requires: py_set_discard_unhashable + +static CYTHON_INLINE int __Pyx_PySet_Discard(PyObject *set, PyObject *key) { + int found = PySet_Discard(set, key); + // Convert *key* to frozenset if necessary + if (unlikely(found < 0)) { + found = __Pyx_PySet_DiscardUnhashable(set, key); + } + // note: returns -1 on error, 0 (not found) or 1 (found) otherwise => error check for -1 or < 0 works + return found; +} + + +/////////////// py_set_remove.proto /////////////// + +static CYTHON_INLINE int __Pyx_PySet_Remove(PyObject *set, PyObject *key); /*proto*/ + +/////////////// py_set_remove /////////////// +//@requires: py_set_discard_unhashable + +static int __Pyx_PySet_RemoveNotFound(PyObject *set, PyObject *key, int found) { + // Convert *key* to frozenset if necessary + if (unlikely(found < 0)) { + found = __Pyx_PySet_DiscardUnhashable(set, key); + } + if (likely(found == 0)) { + // Not found + PyObject *tup; + tup = PyTuple_Pack(1, key); + if (!tup) + return -1; + PyErr_SetObject(PyExc_KeyError, tup); + Py_DECREF(tup); + return -1; + } + // note: returns -1 on error, 0 (not found) or 1 (found) otherwise => error check for -1 or < 0 works + return found; +} + +static CYTHON_INLINE int __Pyx_PySet_Remove(PyObject *set, PyObject *key) { + int found = PySet_Discard(set, key); + if (unlikely(found != 1)) { + // note: returns -1 on error, 0 (not found) or 1 (found) otherwise => error check for -1 or < 0 works + return __Pyx_PySet_RemoveNotFound(set, key, found); + } + return 0; +} + + /////////////// unicode_iter.proto /////////////// static CYTHON_INLINE int __Pyx_init_unicode_iteration( @@ -411,51 +592,366 @@ PyFloat_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #else #define __Pyx_PyObject_AsDouble(obj) \ -((likely(PyFloat_CheckExact(obj))) ? \ - PyFloat_AS_DOUBLE(obj) : __Pyx__PyObject_AsDouble(obj)) +((likely(PyFloat_CheckExact(obj))) ? PyFloat_AS_DOUBLE(obj) : \ + likely(PyLong_CheckExact(obj)) ? \ + PyLong_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #endif /////////////// pyobject_as_double /////////////// +//@requires: pybytes_as_double +//@requires: pyunicode_as_double +//@requires: ObjectHandling.c::PyObjectCallOneArg static double __Pyx__PyObject_AsDouble(PyObject* obj) { - PyObject* float_value; + if (PyUnicode_CheckExact(obj)) { + return __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + return __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + return __Pyx_PyByteArray_AsDouble(obj); + } else { + PyObject* float_value; #if !CYTHON_USE_TYPE_SLOTS - float_value = PyNumber_Float(obj); if (0) goto bad; + float_value = PyNumber_Float(obj); if ((0)) goto bad; + // avoid "unused" warnings + (void)__Pyx_PyObject_CallOneArg; #else - PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; - if (likely(nb) && likely(nb->nb_float)) { - float_value = nb->nb_float(obj); - if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { - PyErr_Format(PyExc_TypeError, - "__float__ returned non-float (type %.200s)", - Py_TYPE(float_value)->tp_name); - Py_DECREF(float_value); - goto bad; + PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; + if (likely(nb) && likely(nb->nb_float)) { + float_value = nb->nb_float(obj); + if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { + __Pyx_TypeName float_value_type_name = __Pyx_PyType_GetName(Py_TYPE(float_value)); + PyErr_Format(PyExc_TypeError, + "__float__ returned non-float (type " __Pyx_FMT_TYPENAME ")", + float_value_type_name); + __Pyx_DECREF_TypeName(float_value_type_name); + Py_DECREF(float_value); + goto bad; + } + } else { + float_value = __Pyx_PyObject_CallOneArg((PyObject*)&PyFloat_Type, obj); } - } else if (PyUnicode_CheckExact(obj) || PyBytes_CheckExact(obj)) { -#if PY_MAJOR_VERSION >= 3 - float_value = PyFloat_FromString(obj); -#else - float_value = PyFloat_FromString(obj, 0); #endif + if (likely(float_value)) { + double value = PyFloat_AS_DOUBLE(float_value); + Py_DECREF(float_value); + return value; + } + } +bad: + return (double)-1; +} + + +/////////////// pystring_as_double.proto /////////////// +//@requires: pyunicode_as_double +//@requires: pybytes_as_double + +static CYTHON_INLINE double __Pyx_PyString_AsDouble(PyObject *obj) { + #if PY_MAJOR_VERSION >= 3 + (void)__Pyx_PyBytes_AsDouble; + return __Pyx_PyUnicode_AsDouble(obj); + #else + (void)__Pyx_PyUnicode_AsDouble; + return __Pyx_PyBytes_AsDouble(obj); + #endif +} + + +/////////////// pyunicode_as_double.proto /////////////// + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj);/*proto*/ + +/////////////// pyunicode_as_double.proto /////////////// +//@requires: pybytes_as_double + +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +static const char* __Pyx__PyUnicode_AsDouble_Copy(const void* data, const int kind, char* buffer, Py_ssize_t start, Py_ssize_t end) { + int last_was_punctuation; + Py_ssize_t i; + // number must not start with punctuation + last_was_punctuation = 1; + for (i=start; i <= end; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + int is_punctuation = (chr == '_') | (chr == '.'); + *buffer = (char)chr; + // reject sequences of '_' and '.' + buffer += (chr != '_'); + if (unlikely(chr > 127)) goto parse_failure; + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyUnicode_AsDouble_inf_nan(const void* data, int kind, Py_ssize_t start, Py_ssize_t length) { + int matches = 1; + Py_UCS4 chr; + Py_UCS4 sign = PyUnicode_READ(kind, data, start); + int is_signed = (sign == '-') | (sign == '+'); + start += is_signed; + length -= is_signed; + + switch (PyUnicode_READ(kind, data, start)) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'a') | (chr == 'A'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'n') | (chr == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'f') | (chr == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+3); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+4); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+5); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+6); + matches &= (chr == 't') | (chr == 'T'); + chr = PyUnicode_READ(kind, data, start+7); + matches &= (chr == 'y') | (chr == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static double __Pyx_PyUnicode_AsDouble_WithSpaces(PyObject *obj) { + double value; + const char *last; + char *end; + Py_ssize_t start, length = PyUnicode_GET_LENGTH(obj); + const int kind = PyUnicode_KIND(obj); + const void* data = PyUnicode_DATA(obj); + + // strip spaces at start and end + start = 0; + while (Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, start))) + start++; + while (start < length - 1 && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, length - 1))) + length--; + length -= start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyUnicode_AsDouble_inf_nan(data, kind, start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + if (length < 40) { + char number[40]; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); } else { - PyObject* args = PyTuple_New(1); - if (unlikely(!args)) goto bad; - PyTuple_SET_ITEM(args, 0, obj); - float_value = PyObject_Call((PyObject*)&PyFloat_Type, args, 0); - PyTuple_SET_ITEM(args, 0, 0); - Py_DECREF(args); + char *number = (char*) PyMem_Malloc((length + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} +#endif + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) { + // Currently not optimised for Py2.7. +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY + if (unlikely(__Pyx_PyUnicode_READY(obj) == -1)) + return (double)-1; + if (likely(PyUnicode_IS_ASCII(obj))) { + const char *s; + Py_ssize_t length; + s = PyUnicode_AsUTF8AndSize(obj, &length); + return __Pyx__PyBytes_AsDouble(obj, s, length); } + return __Pyx_PyUnicode_AsDouble_WithSpaces(obj); +#else + return __Pyx_SlowPyString_AsDouble(obj); +#endif +} + + +/////////////// pybytes_as_double.proto /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj);/*proto*/ +static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length);/*proto*/ + +static CYTHON_INLINE double __Pyx_PyBytes_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); +} +static CYTHON_INLINE double __Pyx_PyByteArray_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); +} + + +/////////////// pybytes_as_double /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj) { + PyObject *float_value; +#if PY_MAJOR_VERSION >= 3 + float_value = PyFloat_FromString(obj); +#else + float_value = PyFloat_FromString(obj, 0); #endif if (likely(float_value)) { double value = PyFloat_AS_DOUBLE(float_value); Py_DECREF(float_value); return value; } -bad: return (double)-1; } +static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) { + // number must not start with punctuation + int last_was_punctuation = 1; + Py_ssize_t i; + for (i=0; i < length; i++) { + char chr = start[i]; + int is_punctuation = (chr == '_') | (chr == '.') | (chr == 'e') | (chr == 'E'); + *buffer = chr; + buffer += (chr != '_'); + // reject sequences of '_' and '.' + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyBytes_AsDouble_inf_nan(const char* start, Py_ssize_t length) { + int matches = 1; + char sign = start[0]; + int is_signed = (sign == '+') | (sign == '-'); + start += is_signed; + length -= is_signed; + + switch (start[0]) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + matches &= (start[1] == 'a' || start[1] == 'A'); + matches &= (start[2] == 'n' || start[2] == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + matches &= (start[1] == 'n' || start[1] == 'N'); + matches &= (start[2] == 'f' || start[2] == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + matches &= (start[3] == 'i' || start[3] == 'I'); + matches &= (start[4] == 'n' || start[4] == 'N'); + matches &= (start[5] == 'i' || start[5] == 'I'); + matches &= (start[6] == 't' || start[6] == 'T'); + matches &= (start[7] == 'y' || start[7] == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static CYTHON_INLINE int __Pyx__PyBytes_AsDouble_IsSpace(char ch) { + // see Py_ISSPACE() in CPython + // https://github.com/python/cpython/blob/master/Python/pyctype.c + return (ch == 0x20) | !((ch < 0x9) | (ch > 0xd)); +} + +static CYTHON_UNUSED double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) { + double value; + Py_ssize_t i, digits; + const char *last = start + length; + char *end; + + // strip spaces at start and end + while (__Pyx__PyBytes_AsDouble_IsSpace(*start)) + start++; + while (start < last - 1 && __Pyx__PyBytes_AsDouble_IsSpace(last[-1])) + last--; + length = last - start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyBytes_AsDouble_inf_nan(start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + // look for underscores + digits = 0; + for (i=0; i < length; digits += start[i++] != '_'); + + if (likely(digits == length)) { + value = PyOS_string_to_double(start, &end, NULL); + } else if (digits < 40) { + char number[40]; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); + } else { + char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} + /////////////// PyNumberPow2.proto /////////////// @@ -468,7 +964,7 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject *none, int inplace) { // in CPython, 1<ob_digit; + if (intval == 0) { + // == 0 => Py_SIZE(pyval) == 0 + {{return_compare('size', '0', c_op)}} + } else if (intval < 0) { + // < 0 => Py_SIZE(pyval) < 0 + if (size >= 0) + {{return_false if op == 'Eq' else return_true}}; + // both are negative => can use absolute values now. + intval = -intval; + size = -size; + } else { + // > 0 => Py_SIZE(pyval) > 0 + if (size <= 0) + {{return_false if op == 'Eq' else return_true}}; + } + // After checking that the sign is the same (and excluding 0), now compare the absolute values. + // When inlining, the C compiler should select exactly one line from this unrolled loop. + uintval = (unsigned long) intval; + {{for _size in range(4, 0, -1)}} +#if PyLong_SHIFT * {{_size}} < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * {{_size}})) { + // The C integer value is between (PyLong_BASE ** _size) and MIN(PyLong_BASE ** _size, LONG_MAX). + unequal = (size != {{_size+1}}) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + {{for _i in range(1, _size+1)}} | (digits[{{_i}}] != ((uintval >> ({{_i}} * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)){{endfor}}; + } else +#endif + {{endfor}} + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + + {{return_compare('unequal', '0', c_op)}} + } + #endif + + if (PyFloat_CheckExact({{pyval}})) { + const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else + double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif + {{return_compare('(double)a', '(double)b', c_op)}} + } + + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyObject_RichCompare(op1, op2, Py_{{op.upper()}})); +} + + /////////////// PyIntBinop.proto /////////////// +{{py: c_ret_type = 'PyObject*' if ret_type.is_pyobject else 'int'}} #if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_{{op}}{{order}}(PyObject *op1, PyObject *op2, long intval, int inplace); /*proto*/ +static {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check); /*proto*/ #else -#define __Pyx_PyInt_{{op}}{{order}}(op1, op2, intval, inplace) \ - {{if op in ('Eq', 'Ne')}}PyObject_RichCompare(op1, op2, Py_{{op.upper()}}) +#define __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(op1, op2, intval, inplace, zerodivision_check) \ + {{if op in ('Eq', 'Ne')}}{{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}(PyObject_RichCompare(op1, op2, Py_{{op.upper()}})) {{else}}(inplace ? PyNumber_InPlace{{op}}(op1, op2) : PyNumber_{{op}}(op1, op2)) {{endif}} #endif @@ -534,19 +1124,36 @@ #if !CYTHON_COMPILING_IN_PYPY {{py: from Cython.Utility import pylong_join }} {{py: pyval, ival = ('op2', 'b') if order == 'CObj' else ('op1', 'a') }} +{{py: c_ret_type = 'PyObject*' if ret_type.is_pyobject else 'int'}} +{{py: return_true = 'Py_RETURN_TRUE' if ret_type.is_pyobject else 'return 1'}} +{{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: slot_name = {'TrueDivide': 'true_divide', 'FloorDivide': 'floor_divide'}.get(op, op.lower()) }} +{{py: cfunc_name = '__Pyx_PyInt_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order)}} {{py: c_op = { - 'Add': '+', 'Subtract': '-', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', + 'Add': '+', 'Subtract': '-', 'Multiply': '*', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', 'Or': '|', 'Xor': '^', 'And': '&', 'Rshift': '>>', 'Lshift': '<<', 'Eq': '==', 'Ne': '!=', }[op] }} +{{py: +def zerodiv_check(operand, optype='integer', _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "%s division%s by zero");' + ' return NULL;' + '}') % (operand, optype, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} + +static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_MAYBE_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); -static PyObject* __Pyx_PyInt_{{op}}{{order}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { {{if op in ('Eq', 'Ne')}} if (op1 == op2) { - Py_RETURN_{{'TRUE' if op == 'Eq' else 'FALSE'}}; + {{return_true if op == 'Eq' else return_false}}; } {{endif}} @@ -557,12 +1164,13 @@ long x; {{endif}} long {{ival}} = PyInt_AS_LONG({{pyval}}); + {{zerodiv_check('b')}} {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { - Py_RETURN_TRUE; + {{return_true}}; } else { - Py_RETURN_FALSE; + {{return_false}}; } {{elif c_op in '+-'}} // adapted from intobject.c in Py2.7: @@ -577,7 +1185,7 @@ x += ((x != 0) & ((x ^ b) < 0)) * b; return PyInt_FromLong(x); {{elif op == 'TrueDivide'}} - if (8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= (1L << 53))) { + if (8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) { return PyFloat_FromDouble((double)a / (double)b); } // let Python do the rounding @@ -596,9 +1204,22 @@ } return PyInt_FromLong(x); {{elif op == 'Lshift'}} - if (likely(a == (a << b) >> b)) { + if (likely(b < (long) (sizeof(long)*8) && a == (a << b) >> b) || !a) { return PyInt_FromLong(a {{c_op}} b); } + {{elif c_op == '*'}} +#ifdef HAVE_LONG_LONG + if (sizeof(PY_LONG_LONG) > sizeof(long)) { + PY_LONG_LONG result = (PY_LONG_LONG)a {{c_op}} (PY_LONG_LONG)b; + return (result >= LONG_MIN && result <= LONG_MAX) ? + PyInt_FromLong((long)result) : PyLong_FromLongLong(result); + } +#endif +#if CYTHON_USE_TYPE_SLOTS + return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); +#else + return PyNumber_{{op}}(op1, op2); +#endif {{else}} // other operations are safe, no overflow return PyInt_FromLong(a {{c_op}} b); @@ -618,6 +1239,38 @@ {{endif}} const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); + {{if c_op == '&'}} + // special case for &-ing arbitrarily large numbers with known single digit operands + if ((intval & PyLong_MASK) == intval) { + long result = 0; + if(likely(size)) { + result = intval & (likely(size>0) ? digits[0] : (PyLong_MASK - digits[0] + 1)); + } + return PyLong_FromLong(result); + } + {{endif}} + // special cases for 0: + - * % / // | ^ & >> << + if (unlikely(size == 0)) { + {{if order == 'CObj' and c_op in '%/'}} + // division by zero! + {{zerodiv_check('0')}} + {{elif order == 'CObj' and c_op in '+-|^>><<'}} + // x == x+0 == x-0 == x|0 == x^0 == x>>0 == x<<0 + return __Pyx_NewRef(op1); + {{elif order == 'CObj' and c_op in '*&'}} + // 0 == x*0 == x&0 + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op in '+|^'}} + // x == 0+x == 0|x == 0^x + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op == '-'}} + // -x == 0-x + return PyLong_FromLong(-intval); + {{elif order == 'ObjC' and (c_op in '*%&>><<' or op == 'FloorDivide')}} + // 0 == 0*x == 0%x == 0&x == 0>>x == 0< {{_size}} * PyLong_SHIFT{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { + if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { {{ival}} = {{'-' if _case < 0 else ''}}(long) {{pylong_join(_size, 'digits')}}; break; {{if op not in ('Eq', 'Ne', 'TrueDivide')}} -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT) { + #ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}) { ll{{ival}} = {{'-' if _case < 0 else ''}}(PY_LONG_LONG) {{pylong_join(_size, 'digits', 'unsigned PY_LONG_LONG')}}; goto long_long; -#endif + #endif {{endif}} } // if size doesn't fit into a long or PY_LONG_LONG anymore, fall through to default + CYTHON_FALLTHROUGH; {{endfor}} {{endfor}} {{if op in ('Eq', 'Ne')}} #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 // unusual setup - your fault - default: return PyLong_Type.tp_richcompare({{'op1, op2' if order == 'ObjC' else 'op2, op1'}}, Py_{{op.upper()}}); + default: return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyLong_Type.tp_richcompare({{'op1, op2' if order == 'ObjC' else 'op2, op1'}}, Py_{{op.upper()}})); #else // too large for the long values we allow => definitely not equal - default: Py_RETURN_{{'FALSE' if op == 'Eq' else 'TRUE'}}; + default: {{return_false if op == 'Eq' else return_true}}; #endif {{else}} default: return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); @@ -657,18 +1312,26 @@ } {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { - Py_RETURN_TRUE; + {{return_true}}; } else { - Py_RETURN_FALSE; + {{return_false}}; } {{else}} - {{if c_op == '%'}} + {{if c_op == '*'}} + (void)a; (void)b; + #ifdef HAVE_LONG_LONG + ll{{ival}} = {{ival}}; + goto long_long; + #else + return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); + #endif + {{elif c_op == '%'}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; {{elif op == 'TrueDivide'}} - if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= (1L << 53))) - || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { + if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) + || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { return PyFloat_FromDouble((double)a / (double)b); } return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); @@ -685,12 +1348,12 @@ x = a {{c_op}} b; {{if op == 'Lshift'}} #ifdef HAVE_LONG_LONG - if (unlikely(a != x >> b)) { + if (unlikely(!(b < (long) (sizeof(long)*8) && a == x >> b)) && a) { ll{{ival}} = {{ival}}; goto long_long; } #else - if (likely(a == x >> b)) /* execute return statement below */ + if (likely(b < (long) (sizeof(long)*8) && a == x >> b) || !a) /* execute return statement below */ #endif {{endif}} {{endif}} @@ -725,18 +1388,23 @@ } #endif - {{if c_op in '+-' or op in ('TrueDivide', 'Eq', 'Ne')}} + {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}} if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{if op in ('Eq', 'Ne')}} if ((double)a {{c_op}} (double)b) { - Py_RETURN_TRUE; + {{return_true}}; } else { - Py_RETURN_FALSE; + {{return_false}}; } {{else}} double result; + {{zerodiv_check('b', 'float')}} // copied from floatobject.c in Py3.5: PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; @@ -747,7 +1415,8 @@ {{endif}} {{if op in ('Eq', 'Ne')}} - return PyObject_RichCompare(op1, op2, Py_{{op.upper()}}); + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyObject_RichCompare(op1, op2, Py_{{op.upper()}})); {{else}} return (inplace ? PyNumber_InPlace{{op}} : PyNumber_{{op}})(op1, op2); {{endif}} @@ -756,11 +1425,12 @@ /////////////// PyFloatBinop.proto /////////////// +{{py: c_ret_type = 'PyObject*' if ret_type.is_pyobject else 'int'}} #if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyFloat_{{op}}{{order}}(PyObject *op1, PyObject *op2, double floatval, int inplace); /*proto*/ +static {{c_ret_type}} __Pyx_PyFloat_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, double floatval, int inplace, int zerodivision_check); /*proto*/ #else -#define __Pyx_PyFloat_{{op}}{{order}}(op1, op2, floatval, inplace) \ - {{if op in ('Eq', 'Ne')}}PyObject_RichCompare(op1, op2, Py_{{op.upper()}}) +#define __Pyx_PyFloat_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(op1, op2, floatval, inplace, zerodivision_check) \ + {{if op in ('Eq', 'Ne')}}{{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}(PyObject_RichCompare(op1, op2, Py_{{op.upper()}})) {{elif op == 'Divide'}}((inplace ? __Pyx_PyNumber_InPlaceDivide(op1, op2) : __Pyx_PyNumber_Divide(op1, op2))) {{else}}(inplace ? PyNumber_InPlace{{op}}(op1, op2) : PyNumber_{{op}}(op1, op2)) {{endif}} @@ -770,31 +1440,52 @@ #if !CYTHON_COMPILING_IN_PYPY {{py: from Cython.Utility import pylong_join }} +{{py: c_ret_type = 'PyObject*' if ret_type.is_pyobject else 'int'}} +{{py: return_true = 'Py_RETURN_TRUE' if ret_type.is_pyobject else 'return 1'}} +{{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: pyval, fval = ('op2', 'b') if order == 'CObj' else ('op1', 'a') }} +{{py: cfunc_name = '__Pyx_PyFloat_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order) }} {{py: c_op = { 'Add': '+', 'Subtract': '-', 'TrueDivide': '/', 'Divide': '/', 'Remainder': '%', 'Eq': '==', 'Ne': '!=', }[op] }} +{{py: +def zerodiv_check(operand, _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0.0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "float division%s by zero");' + ' return NULL;' + '}') % (operand, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} -static PyObject* __Pyx_PyFloat_{{op}}{{order}}(PyObject *op1, PyObject *op2, double floatval, CYTHON_UNUSED int inplace) { +static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatval, int inplace, int zerodivision_check) { const double {{'a' if order == 'CObj' else 'b'}} = floatval; double {{fval}}{{if op not in ('Eq', 'Ne')}}, result{{endif}}; + // Prevent "unused" warnings. + (void)inplace; (void)zerodivision_check; {{if op in ('Eq', 'Ne')}} if (op1 == op2) { - Py_RETURN_{{'TRUE' if op == 'Eq' else 'FALSE'}}; + {{return_true if op == 'Eq' else return_false}}; } {{endif}} if (likely(PyFloat_CheckExact({{pyval}}))) { +#if CYTHON_COMPILING_IN_LIMITED_API + {{fval}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else {{fval}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif + {{zerodiv_check(fval)}} } else #if PY_MAJOR_VERSION < 3 if (likely(PyInt_CheckExact({{pyval}}))) { {{fval}} = (double) PyInt_AS_LONG({{pyval}}); + {{zerodiv_check(fval)}} } else #endif @@ -803,7 +1494,7 @@ const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); switch (size) { - case 0: {{fval}} = 0.0; break; + case 0: {{fval}} = 0.0; {{zerodiv_check(fval)}} break; case -1: {{fval}} = -(double) digits[0]; break; case 1: {{fval}} = (double) digits[0]; break; {{for _size in (2, 3, 4)}} @@ -812,7 +1503,7 @@ if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) { {{fval}} = (double) {{pylong_join(_size, 'digits')}}; // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float) - if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) (1L<<53))) { + if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) { if (size == {{-_size}}) {{fval}} = -{{fval}}; break; @@ -823,21 +1514,29 @@ // check above. However, the number of digits that CPython uses for a given PyLong // value is minimal, and together with the "(size-1) * SHIFT < 53" check above, // this should make it safe. + CYTHON_FALLTHROUGH; {{endfor}} default: - #else - { #endif {{if op in ('Eq', 'Ne')}} - return PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}}); + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}})); {{else}} {{fval}} = PyLong_AsDouble({{pyval}}); if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; + {{if zerodiv_check(fval)}} + #if !CYTHON_USE_PYLONG_INTERNALS + {{zerodiv_check(fval)}} + #endif + {{endif}} {{endif}} + #if CYTHON_USE_PYLONG_INTERNALS } + #endif } else { {{if op in ('Eq', 'Ne')}} - return PyObject_RichCompare(op1, op2, Py_{{op.upper()}}); + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyObject_RichCompare(op1, op2, Py_{{op.upper()}})); {{elif op == 'Divide'}} return (inplace ? __Pyx_PyNumber_InPlaceDivide(op1, op2) : __Pyx_PyNumber_Divide(op1, op2)); {{else}} @@ -847,9 +1546,9 @@ {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { - Py_RETURN_TRUE; + {{return_true}}; } else { - Py_RETURN_FALSE; + {{return_false}}; } {{else}} // copied from floatobject.c in Py3.5: diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Overflow.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Overflow.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Overflow.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Overflow.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,7 @@ /* These functions provide integer arithmetic with integer checking. They do not actually raise an exception when an overflow is detected, but rather set a bit -in the overflow parameter. (This parameter may be re-used accross several +in the overflow parameter. (This parameter may be re-used across several arithmetic operations, so should be or-ed rather than assigned to.) The implementation is divided into two parts, the signed and unsigned basecases, @@ -20,10 +20,10 @@ /////////////// Common.proto /////////////// static int __Pyx_check_twos_complement(void) { - if (-1 != ~0) { + if ((-1 != ~0)) { PyErr_SetString(PyExc_RuntimeError, "Two's complement required for overflow checks."); return 1; - } else if (sizeof(short) == sizeof(int)) { + } else if ((sizeof(short) == sizeof(int))) { PyErr_SetString(PyExc_RuntimeError, "sizeof(short) < sizeof(int) required for overflow checks."); return 1; } else { @@ -31,11 +31,11 @@ } } -#define __PYX_IS_UNSIGNED(type) (((type) -1) > 0) -#define __PYX_SIGN_BIT(type) (((unsigned type) 1) << (sizeof(type) * 8 - 1)) -#define __PYX_HALF_MAX(type) (((type) 1) << (sizeof(type) * 8 - 2)) -#define __PYX_MIN(type) (__PYX_IS_UNSIGNED(type) ? (type) 0 : 0 - __PYX_HALF_MAX(type) - __PYX_HALF_MAX(type)) -#define __PYX_MAX(type) (~__PYX_MIN(type)) +#define __PYX_IS_UNSIGNED(type) ((((type) -1) > 0)) +#define __PYX_SIGN_BIT(type) ((((unsigned type) 1) << (sizeof(type) * 8 - 1))) +#define __PYX_HALF_MAX(type) ((((type) 1) << (sizeof(type) * 8 - 2))) +#define __PYX_MIN(type) ((__PYX_IS_UNSIGNED(type) ? (type) 0 : 0 - __PYX_HALF_MAX(type) - __PYX_HALF_MAX(type))) +#define __PYX_MAX(type) ((~__PYX_MIN(type))) #define __Pyx_add_no_overflow(a, b, overflow) ((a) + (b)) #define __Pyx_add_const_no_overflow(a, b, overflow) ((a) + (b)) @@ -46,12 +46,36 @@ #define __Pyx_div_no_overflow(a, b, overflow) ((a) / (b)) #define __Pyx_div_const_no_overflow(a, b, overflow) ((a) / (b)) +#if defined(__has_builtin) +# if __has_builtin(__builtin_add_overflow) && !defined(__ibmxl__) +# define __PYX_HAVE_BUILTIN_OVERFLOW +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) && (!defined(__INTEL_COMPILER) || (__INTEL_COMPILER >= 1800)) +# define __PYX_HAVE_BUILTIN_OVERFLOW +#endif + +#if defined(__GNUC__) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +#elif defined(__has_builtin) +# if __has_builtin(__builtin_constant_p) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +# endif +#else +# define __Pyx_is_constant(x) (0) +#endif + /////////////// Common.init /////////////// +//@substitute: naming -__Pyx_check_twos_complement(); +// FIXME: Propagate the error here instead of just printing it. +if (unlikely(__Pyx_check_twos_complement())) { + PyErr_WriteUnraisable($module_cname); +} /////////////// BaseCaseUnsigned.proto /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); @@ -60,11 +84,41 @@ // Use these when b is known at compile time. #define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow #define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseUnsigned /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { {{UINT}} r = a + b; *overflow |= r < a; @@ -78,34 +132,45 @@ } static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if (sizeof({{UINT}}) < sizeof(unsigned long)) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{UINT}}) < sizeof(unsigned long))) { unsigned long big_r = ((unsigned long) a) * ((unsigned long) b); {{UINT}} r = ({{UINT}}) big_r; *overflow |= big_r != r; return r; #ifdef HAVE_LONG_LONG - } else if (sizeof({{UINT}}) < sizeof(unsigned PY_LONG_LONG)) { + } else if ((sizeof({{UINT}}) < sizeof(unsigned PY_LONG_LONG))) { unsigned PY_LONG_LONG big_r = ((unsigned PY_LONG_LONG) a) * ((unsigned PY_LONG_LONG) b); {{UINT}} r = ({{UINT}}) big_r; *overflow |= big_r != r; return r; #endif } else { - {{UINT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(UINT), - // whereas rounding represents an error on the order of 2^(sizeof(UINT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{UINT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if (b > 1) { - *overflow |= a > __PYX_MAX({{UINT}}) / b; - } - return a * b; + // note that deliberately the overflow check is written such that it divides by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operation + {{UINT}} prod; + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{UINT}} temp = b; + b = a; + a = temp; + } + prod = a * b; + if (b != 0) + *overflow |= a > (__PYX_MAX({{UINT}}) / b); + return prod; } +#endif // __PYX_HAVE_BUILTIN_OVERFLOW static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { @@ -116,9 +181,13 @@ return a / b; } +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned.proto /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); @@ -126,83 +195,110 @@ // Use when b is known at compile time. -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); +#define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow +#define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if INT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (sizeof({{INT}}) < sizeof(long)) { + if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) + ((long) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; return r; #ifdef HAVE_LONG_LONG - } else if (sizeof({{INT}}) < sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{INT}}) < sizeof(PY_LONG_LONG))) { PY_LONG_LONG big_r = ((PY_LONG_LONG) a) + ((PY_LONG_LONG) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; return r; #endif } else { - // Signed overflow undefined, but unsigned overflow is well defined. - {{INT}} r = ({{INT}}) ((unsigned {{INT}}) a + (unsigned {{INT}}) b); + // Signed overflow undefined, but unsigned overflow is well defined. Casting is + // implementation-defined, but we assume two's complement (see __Pyx_check_twos_complement + // above), and arithmetic in two's-complement is the same as unsigned arithmetic. + unsigned {{INT}} r = (unsigned {{INT}}) a + (unsigned {{INT}}) b; // Overflow happened if the operands have the same sign, but the result // has opposite sign. - // sign(a) == sign(b) != sign(r) - {{INT}} sign_a = __PYX_SIGN_BIT({{INT}}) & a; - {{INT}} sign_b = __PYX_SIGN_BIT({{INT}}) & b; - {{INT}} sign_r = __PYX_SIGN_BIT({{INT}}) & r; - *overflow |= (sign_a == sign_b) & (sign_a != sign_r); - return r; - } -} - -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (b > 0) { - *overflow |= a > __PYX_MAX({{INT}}) - b; - } else if (b < 0) { - *overflow |= a < __PYX_MIN({{INT}}) - b; + *overflow |= (((unsigned {{INT}})a ^ r) & ((unsigned {{INT}})b ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } - return a + b; } static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_{{NAME}}_checking_overflow(a, -b, overflow); -} - -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_const_{{NAME}}_checking_overflow(a, -b, overflow); + // Compilers don't handle widening as well in the subtraction case, so don't bother + unsigned {{INT}} r = (unsigned {{INT}}) a - (unsigned {{INT}}) b; + // Overflow happened if the operands differing signs, and the result + // has opposite sign to a. + *overflow |= (((unsigned {{INT}})a ^ (unsigned {{INT}})b) & ((unsigned {{INT}})a ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (sizeof({{INT}}) < sizeof(long)) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) * ((long) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; return ({{INT}}) r; #ifdef HAVE_LONG_LONG - } else if (sizeof({{INT}}) < sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{INT}}) < sizeof(PY_LONG_LONG))) { PY_LONG_LONG big_r = ((PY_LONG_LONG) a) * ((PY_LONG_LONG) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; return ({{INT}}) r; #endif } else { - {{INT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(INT), - // whereas rounding represents an error on the order of 2^(sizeof(INT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{INT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + // note that deliberately all these comparisons are written such that they divide by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operations + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{INT}} temp = b; + b = a; + a = temp; + } if (b > 1) { *overflow |= a > __PYX_MAX({{INT}}) / b; *overflow |= a < __PYX_MIN({{INT}}) / b; @@ -212,31 +308,38 @@ *overflow |= a > __PYX_MIN({{INT}}) / b; *overflow |= a < __PYX_MAX({{INT}}) / b; } - return a * b; + return ({{INT}}) (((unsigned {{INT}})a) * ((unsigned {{INT}}) b)); } +#endif // defined(__PYX_HAVE_BUILTIN_OVERFLOW) static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if (b == 0) { *overflow |= 1; return 0; } - *overflow |= (a == __PYX_MIN({{INT}})) & (b == -1); - return a / b; + *overflow |= a == __PYX_MIN({{INT}}) && b == -1; + return ({{INT}}) ((unsigned {{INT}}) a / (unsigned {{INT}}) b); } +{{if INT == "long long"}}#endif{{endif}} + /////////////// SizeCheck.init /////////////// +//@substitute: naming -__Pyx_check_sane_{{NAME}}(); +// FIXME: Propagate the error here instead of just printing it. +if (unlikely(__Pyx_check_sane_{{NAME}}())) { + PyErr_WriteUnraisable($module_cname); +} /////////////// SizeCheck.proto /////////////// static int __Pyx_check_sane_{{NAME}}(void) { - if (sizeof({{TYPE}}) <= sizeof(int) || + if (((sizeof({{TYPE}}) <= sizeof(int)) || #ifdef HAVE_LONG_LONG - sizeof({{TYPE}}) == sizeof(PY_LONG_LONG) || + (sizeof({{TYPE}}) == sizeof(PY_LONG_LONG)) || #endif - sizeof({{TYPE}}) == sizeof(long)) { + (sizeof({{TYPE}}) == sizeof(long)))) { return 0; } else { PyErr_Format(PyExc_RuntimeError, \ @@ -253,31 +356,31 @@ /////////////// Binop /////////////// static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE}} a, {{TYPE}} b, int *overflow) { - if (sizeof({{TYPE}}) < sizeof(int)) { + if ((sizeof({{TYPE}}) < sizeof(int))) { return __Pyx_{{BINOP}}_no_overflow(a, b, overflow); } else if (__PYX_IS_UNSIGNED({{TYPE}})) { - if (sizeof({{TYPE}}) == sizeof(unsigned int)) { - return __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); - } else if (sizeof({{TYPE}}) == sizeof(unsigned long)) { - return __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); + if ((sizeof({{TYPE}}) == sizeof(unsigned int))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); + } else if ((sizeof({{TYPE}}) == sizeof(unsigned long))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) == sizeof(unsigned PY_LONG_LONG)) { - return __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); + } else if ((sizeof({{TYPE}}) == sizeof(unsigned PY_LONG_LONG))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); #endif } else { - abort(); return 0; // handled elsewhere + abort(); return 0; /* handled elsewhere */ } } else { - if (sizeof({{TYPE}}) == sizeof(int)) { - return __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); - } else if (sizeof({{TYPE}}) == sizeof(long)) { - return __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); + if ((sizeof({{TYPE}}) == sizeof(int))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); + } else if ((sizeof({{TYPE}}) == sizeof(long))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) == sizeof(PY_LONG_LONG)) { - return __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); + } else if ((sizeof({{TYPE}}) == sizeof(PY_LONG_LONG))) { + return ({{TYPE}}) __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); #endif } else { - abort(); return 0; // handled elsewhere + abort(); return 0; /* handled elsewhere */ } } } @@ -285,19 +388,24 @@ /////////////// LeftShift.proto /////////////// static CYTHON_INLINE {{TYPE}} __Pyx_lshift_{{NAME}}_checking_overflow({{TYPE}} a, {{TYPE}} b, int *overflow) { - *overflow |= + int overflow_check = #if {{SIGNED}} - (b < 0) | + (a < 0) || (b < 0) || #endif - (b > ({{TYPE}}) (8 * sizeof({{TYPE}}))) | (a > (__PYX_MAX({{TYPE}}) >> b)); - return a << b; + // the following must be a _logical_ OR as the RHS is undefined if the LHS is true + (b >= ({{TYPE}}) (8 * sizeof({{TYPE}}))) || (a > (__PYX_MAX({{TYPE}}) >> b)); + if (overflow_check) { + *overflow |= 1; + return 0; + } else { + return a << b; + } } #define __Pyx_lshift_const_{{NAME}}_checking_overflow __Pyx_lshift_{{NAME}}_checking_overflow /////////////// UnaryNegOverflows.proto /////////////// -//FIXME: shouldn't the macro name be prefixed by "__Pyx_" ? Too late now, I guess... // from intobject.c -#define UNARY_NEG_WOULD_OVERFLOW(x) \ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x) \ (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/Profile.c cython-0.20.1+1~202203241016-9537/Cython/Utility/Profile.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/Profile.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/Profile.c 2022-03-24 10:16:46.000000000 +0000 @@ -1,11 +1,12 @@ /////////////// Profile.proto /////////////// +//@requires: Exceptions.c::PyErrFetchRestore //@substitute: naming // Note that cPython ignores PyTrace_EXCEPTION, // but maybe some other profilers don't. #ifndef CYTHON_PROFILE -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY #define CYTHON_PROFILE 0 #else #define CYTHON_PROFILE 1 @@ -37,6 +38,12 @@ #include "compile.h" #include "frameobject.h" #include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif #if CYTHON_PROFILE_REUSE_FRAME #define CYTHON_FRAME_MODIFIER static @@ -46,13 +53,57 @@ #define CYTHON_FRAME_DEL(frame) Py_CLEAR(frame) #endif - #define __Pyx_TraceDeclarations \ - static PyCodeObject *$frame_code_cname = NULL; \ - CYTHON_FRAME_MODIFIER PyFrameObject *$frame_cname = NULL; \ - int __Pyx_use_tracing = 0; + #define __Pyx_TraceDeclarations \ + static PyCodeObject *$frame_code_cname = NULL; \ + CYTHON_FRAME_MODIFIER PyFrameObject *$frame_cname = NULL; \ + int __Pyx_use_tracing = 0; + + #define __Pyx_TraceFrameInit(codeobj) \ + if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj; + +#if PY_VERSION_HEX >= 0x030b00a2 + #define __Pyx_IsTracing(tstate, check_tracing, check_funcs) \ + (unlikely((tstate)->cframe->use_tracing) && \ + (!(check_tracing) || !(tstate)->tracing) && \ + (!(check_funcs) || (tstate)->c_profilefunc || (CYTHON_TRACE && (tstate)->c_tracefunc))) + + #define __Pyx_EnterTracing(tstate) PyThreadState_EnterTracing(tstate) + + #define __Pyx_LeaveTracing(tstate) PyThreadState_LeaveTracing(tstate) + +#elif PY_VERSION_HEX >= 0x030a00b1 + #define __Pyx_IsTracing(tstate, check_tracing, check_funcs) \ + (unlikely((tstate)->cframe->use_tracing) && \ + (!(check_tracing) || !(tstate)->tracing) && \ + (!(check_funcs) || (tstate)->c_profilefunc || (CYTHON_TRACE && (tstate)->c_tracefunc))) + + #define __Pyx_EnterTracing(tstate) \ + do { tstate->tracing++; tstate->cframe->use_tracing = 0; } while (0) + + #define __Pyx_LeaveTracing(tstate) \ + do { \ + tstate->tracing--; \ + tstate->cframe->use_tracing = ((CYTHON_TRACE && tstate->c_tracefunc != NULL) \ + || tstate->c_profilefunc != NULL); \ + } while (0) - #define __Pyx_TraceFrameInit(codeobj) \ - if (codeobj) $frame_code_cname = (PyCodeObject*) codeobj; +#else + #define __Pyx_IsTracing(tstate, check_tracing, check_funcs) \ + (unlikely((tstate)->use_tracing) && \ + (!(check_tracing) || !(tstate)->tracing) && \ + (!(check_funcs) || (tstate)->c_profilefunc || (CYTHON_TRACE && (tstate)->c_tracefunc))) + + #define __Pyx_EnterTracing(tstate) \ + do { tstate->tracing++; tstate->use_tracing = 0; } while (0) + + #define __Pyx_LeaveTracing(tstate) \ + do { \ + tstate->tracing--; \ + tstate->use_tracing = ((CYTHON_TRACE && tstate->c_tracefunc != NULL) \ + || tstate->c_profilefunc != NULL); \ + } while (0) + +#endif #ifdef WITH_THREAD #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) \ @@ -60,28 +111,25 @@ if (CYTHON_TRACE_NOGIL) { \ PyThreadState *tstate; \ PyGILState_STATE state = PyGILState_Ensure(); \ - tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing) && !tstate->tracing && \ - (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \ + tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 1, 1)) { \ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, tstate, funcname, srcfile, firstlineno); \ } \ PyGILState_Release(state); \ if (unlikely(__Pyx_use_tracing < 0)) goto_error; \ } \ } else { \ PyThreadState* tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing) && !tstate->tracing && \ - (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \ + if (__Pyx_IsTracing(tstate, 1, 1)) { \ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, tstate, funcname, srcfile, firstlineno); \ if (unlikely(__Pyx_use_tracing < 0)) goto_error; \ } \ } #else #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) \ { PyThreadState* tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing) && !tstate->tracing && \ - (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, funcname, srcfile, firstlineno); \ + if (__Pyx_IsTracing(tstate, 1, 1)) { \ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&$frame_code_cname, &$frame_cname, tstate, funcname, srcfile, firstlineno); \ if (unlikely(__Pyx_use_tracing < 0)) goto_error; \ } \ } @@ -89,11 +137,9 @@ #define __Pyx_TraceException() \ if (likely(!__Pyx_use_tracing)); else { \ - PyThreadState* tstate = PyThreadState_GET(); \ - if (tstate->use_tracing && \ - (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) { \ - tstate->tracing++; \ - tstate->use_tracing = 0; \ + PyThreadState* tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 1)) { \ + __Pyx_EnterTracing(tstate); \ PyObject *exc_info = __Pyx_GetExceptionTuple(tstate); \ if (exc_info) { \ if (CYTHON_TRACE && tstate->c_tracefunc) \ @@ -103,24 +149,21 @@ tstate->c_profileobj, $frame_cname, PyTrace_EXCEPTION, exc_info); \ Py_DECREF(exc_info); \ } \ - tstate->use_tracing = 1; \ - tstate->tracing--; \ + __Pyx_LeaveTracing(tstate); \ } \ } static void __Pyx_call_return_trace_func(PyThreadState *tstate, PyFrameObject *frame, PyObject *result) { PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - tstate->tracing++; - tstate->use_tracing = 0; + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); + __Pyx_EnterTracing(tstate); if (CYTHON_TRACE && tstate->c_tracefunc) tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_RETURN, result); if (tstate->c_profilefunc) tstate->c_profilefunc(tstate->c_profileobj, frame, PyTrace_RETURN, result); CYTHON_FRAME_DEL(frame); - tstate->use_tracing = 1; - tstate->tracing--; - PyErr_Restore(type, value, traceback); + __Pyx_LeaveTracing(tstate); + __Pyx_ErrRestoreInState(tstate, type, value, traceback); } #ifdef WITH_THREAD @@ -130,15 +173,15 @@ if (CYTHON_TRACE_NOGIL) { \ PyThreadState *tstate; \ PyGILState_STATE state = PyGILState_Ensure(); \ - tstate = PyThreadState_GET(); \ - if (tstate->use_tracing) { \ + tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0)) { \ __Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \ } \ PyGILState_Release(state); \ } \ } else { \ - PyThreadState* tstate = PyThreadState_GET(); \ - if (tstate->use_tracing) { \ + PyThreadState* tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0)) { \ __Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \ } \ } \ @@ -146,22 +189,22 @@ #else #define __Pyx_TraceReturn(result, nogil) \ if (likely(!__Pyx_use_tracing)); else { \ - PyThreadState* tstate = PyThreadState_GET(); \ - if (tstate->use_tracing) { \ + PyThreadState* tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0)) { \ __Pyx_call_return_trace_func(tstate, $frame_cname, (PyObject*)result); \ } \ } #endif static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); /*proto*/ - static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); /*proto*/ + static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, PyThreadState* tstate, const char *funcname, const char *srcfile, int firstlineno); /*proto*/ #else #define __Pyx_TraceDeclarations #define __Pyx_TraceFrameInit(codeobj) // mark error label as used to avoid compiler warnings - #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) if (1); else goto_error; + #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) if ((1)); else goto_error; #define __Pyx_TraceException() #define __Pyx_TraceReturn(result, nogil) @@ -172,15 +215,15 @@ static int __Pyx_call_line_trace_func(PyThreadState *tstate, PyFrameObject *frame, int lineno) { int ret; PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); __Pyx_PyFrame_SetLineNumber(frame, lineno); - tstate->tracing++; - tstate->use_tracing = 0; + __Pyx_EnterTracing(tstate); + ret = tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_LINE, NULL); - tstate->use_tracing = 1; - tstate->tracing--; + + __Pyx_LeaveTracing(tstate); if (likely(!ret)) { - PyErr_Restore(type, value, traceback); + __Pyx_ErrRestoreInState(tstate, type, value, traceback); } else { Py_XDECREF(type); Py_XDECREF(value); @@ -196,17 +239,17 @@ if (CYTHON_TRACE_NOGIL) { \ int ret = 0; \ PyThreadState *tstate; \ - PyGILState_STATE state = PyGILState_Ensure(); \ - tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \ + PyGILState_STATE state = __Pyx_PyGILState_Ensure(); \ + tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0) && tstate->c_tracefunc && $frame_cname->f_trace) { \ ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ } \ - PyGILState_Release(state); \ + __Pyx_PyGILState_Release(state); \ if (unlikely(ret)) goto_error; \ } \ } else { \ - PyThreadState* tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \ + PyThreadState* tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0) && tstate->c_tracefunc && $frame_cname->f_trace) { \ int ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ if (unlikely(ret)) goto_error; \ } \ @@ -215,8 +258,8 @@ #else #define __Pyx_TraceLine(lineno, nogil, goto_error) \ if (likely(!__Pyx_use_tracing)); else { \ - PyThreadState* tstate = PyThreadState_GET(); \ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) { \ + PyThreadState* tstate = __Pyx_PyThreadState_Current; \ + if (__Pyx_IsTracing(tstate, 0, 0) && tstate->c_tracefunc && $frame_cname->f_trace) { \ int ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ if (unlikely(ret)) goto_error; \ } \ @@ -224,7 +267,7 @@ #endif #else // mark error label as used to avoid compiler warnings - #define __Pyx_TraceLine(lineno, nogil, goto_error) if (1); else goto_error; + #define __Pyx_TraceLine(lineno, nogil, goto_error) if ((1)); else goto_error; #endif /////////////// Profile /////////////// @@ -234,12 +277,12 @@ static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, + PyThreadState* tstate, const char *funcname, const char *srcfile, int firstlineno) { PyObject *type, *value, *traceback; int retval; - PyThreadState* tstate = PyThreadState_GET(); if (*frame == NULL || !CYTHON_PROFILE_REUSE_FRAME) { if (*code == NULL) { *code = __Pyx_createFrameCodeObject(funcname, srcfile, firstlineno); @@ -262,23 +305,23 @@ (*frame)->f_tstate = tstate; #endif } - __Pyx_PyFrame_SetLineNumber(*frame, firstlineno); + __Pyx_PyFrame_SetLineNumber(*frame, firstlineno); + retval = 1; - tstate->tracing++; - tstate->use_tracing = 0; - PyErr_Fetch(&type, &value, &traceback); + __Pyx_EnterTracing(tstate); + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); + #if CYTHON_TRACE if (tstate->c_tracefunc) retval = tstate->c_tracefunc(tstate->c_traceobj, *frame, PyTrace_CALL, NULL) == 0; if (retval && tstate->c_profilefunc) #endif retval = tstate->c_profilefunc(tstate->c_profileobj, *frame, PyTrace_CALL, NULL) == 0; - tstate->use_tracing = (tstate->c_profilefunc || - (CYTHON_TRACE && tstate->c_tracefunc)); - tstate->tracing--; + + __Pyx_LeaveTracing(tstate); if (retval) { - PyErr_Restore(type, value, traceback); - return tstate->use_tracing && retval; + __Pyx_ErrRestoreInState(tstate, type, value, traceback); + return __Pyx_IsTracing(tstate, 0, 0) && retval; } else { Py_XDECREF(type); Py_XDECREF(value); @@ -288,27 +331,29 @@ } static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno) { + PyCodeObject *py_code = 0; + +#if PY_MAJOR_VERSION >= 3 + py_code = PyCode_NewEmpty(srcfile, funcname, firstlineno); + // make CPython use a fresh dict for "f_locals" at need (see GH #1836) + if (likely(py_code)) { + py_code->co_flags |= CO_OPTIMIZED | CO_NEWLOCALS; + } +#else PyObject *py_srcfile = 0; PyObject *py_funcname = 0; - PyCodeObject *py_code = 0; - #if PY_MAJOR_VERSION < 3 py_funcname = PyString_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; py_srcfile = PyString_FromString(srcfile); - #else - py_funcname = PyUnicode_FromString(funcname); - py_srcfile = PyUnicode_FromString(srcfile); - #endif - if (!py_funcname | !py_srcfile) goto bad; + if (unlikely(!py_srcfile)) goto bad; py_code = PyCode_New( 0, /*int argcount,*/ - #if PY_MAJOR_VERSION >= 3 - 0, /*int kwonlyargcount,*/ - #endif 0, /*int nlocals,*/ 0, /*int stacksize,*/ - 0, /*int flags,*/ + // make CPython use a fresh dict for "f_locals" at need (see GH #1836) + CO_OPTIMIZED | CO_NEWLOCALS, /*int flags,*/ $empty_bytes, /*PyObject *code,*/ $empty_tuple, /*PyObject *consts,*/ $empty_tuple, /*PyObject *names,*/ @@ -324,6 +369,7 @@ bad: Py_XDECREF(py_srcfile); Py_XDECREF(py_funcname); +#endif return py_code; } diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/StringTools.c cython-0.20.1+1~202203241016-9537/Cython/Utility/StringTools.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/StringTools.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/StringTools.c 2022-03-24 10:16:46.000000000 +0000 @@ -7,15 +7,78 @@ #include + +//////////////////// ssize_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);/*proto*/ + +//////////////////// ssize_strlen //////////////////// +//@requires: IncludeStringH + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + +//////////////////// ssize_pyunicode_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u);/*proto*/ + +//////////////////// ssize_pyunicode_strlen //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u) { + size_t len = __Pyx_Py_UNICODE_strlen(u); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "Py_UNICODE string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + //////////////////// InitStrings.proto //////////////////// +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str); /*proto*/ +#else static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ +#endif //////////////////// InitStrings //////////////////// +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) { + if (t.is_unicode | t.is_str) { + if (t.intern) { + *str = PyUnicode_InternFromString(t.s); + } else if (t.encoding) { + *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL); + } else { + *str = PyUnicode_FromStringAndSize(t.s, t.n - 1); + } + } else { + *str = PyBytes_FromStringAndSize(t.s, t.n - 1); + } + if (!*str) + return -1; + // initialise cached hash value + if (PyObject_Hash(*str) == -1) + return -1; + return 0; +} +#endif + +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION >= 3 /* Python 3+ has unicode identifiers */ + __Pyx_InitString(*t, t->p); + #else if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { @@ -23,25 +86,17 @@ } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } - #else /* Python 3+ has unicode identifiers */ - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif if (!*t->p) return -1; + // initialise cached hash value + if (PyObject_Hash(*t->p) == -1) + return -1; + #endif ++t; } return 0; } +#endif //////////////////// BytesContains.proto //////////////////// @@ -60,14 +115,53 @@ //////////////////// PyUCS4InUnicode.proto //////////////////// static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character); /*proto*/ -static CYTHON_INLINE int __Pyx_PyUnicodeBufferContainsUCS4(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character); /*proto*/ //////////////////// PyUCS4InUnicode //////////////////// +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) + +#if PY_VERSION_HEX < 0x03090000 +#define __Pyx_PyUnicode_AS_UNICODE(op) PyUnicode_AS_UNICODE(op) +#define __Pyx_PyUnicode_GET_SIZE(op) PyUnicode_GET_SIZE(op) +#else +// Avoid calling deprecated C-API functions in Py3.9+ that PEP-623 schedules for removal in Py3.12. +// https://www.python.org/dev/peps/pep-0623/ +#define __Pyx_PyUnicode_AS_UNICODE(op) (((PyASCIIObject *)(op))->wstr) +#define __Pyx_PyUnicode_GET_SIZE(op) ((PyCompactUnicodeObject *)(op))->wstr_length +#endif + +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 +static int __Pyx_PyUnicodeBufferContainsUCS4_SP(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character) { + /* handle surrogate pairs for Py_UNICODE buffers in 16bit Unicode builds */ + Py_UNICODE high_val, low_val; + Py_UNICODE* pos; + high_val = (Py_UNICODE) (0xD800 | (((character - 0x10000) >> 10) & ((1<<10)-1))); + low_val = (Py_UNICODE) (0xDC00 | ( (character - 0x10000) & ((1<<10)-1))); + for (pos=buffer; pos < buffer+length-1; pos++) { + if (unlikely((high_val == pos[0]) & (low_val == pos[1]))) return 1; + } + return 0; +} +#endif + +static int __Pyx_PyUnicodeBufferContainsUCS4_BMP(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character) { + Py_UNICODE uchar; + Py_UNICODE* pos; + uchar = (Py_UNICODE) character; + for (pos=buffer; pos < buffer+length; pos++) { + if (unlikely(uchar == pos[0])) return 1; + } + return 0; +} +#endif + static CYTHON_INLINE int __Pyx_UnicodeContainsUCS4(PyObject* unicode, Py_UCS4 character) { #if CYTHON_PEP393_ENABLED const int kind = PyUnicode_KIND(unicode); - if (likely(kind != PyUnicode_WCHAR_KIND)) { + #ifdef PyUnicode_WCHAR_KIND + if (likely(kind != PyUnicode_WCHAR_KIND)) + #endif + { Py_ssize_t i; const void* udata = PyUnicode_DATA(unicode); const Py_ssize_t length = PyUnicode_GET_LENGTH(unicode); @@ -76,33 +170,29 @@ } return 0; } +#elif PY_VERSION_HEX >= 0x03090000 + #error Cannot use "UChar in Unicode" in Python 3.9 without PEP-393 unicode strings. +#elif !defined(PyUnicode_AS_UNICODE) + #error Cannot use "UChar in Unicode" in Python < 3.9 without Py_UNICODE support. #endif - return __Pyx_PyUnicodeBufferContainsUCS4( - PyUnicode_AS_UNICODE(unicode), - PyUnicode_GET_SIZE(unicode), - character); -} -static CYTHON_INLINE int __Pyx_PyUnicodeBufferContainsUCS4(Py_UNICODE* buffer, Py_ssize_t length, Py_UCS4 character) { - Py_UNICODE uchar; - Py_UNICODE* pos; - #if Py_UNICODE_SIZE == 2 - if (character > 65535) { - /* handle surrogate pairs for Py_UNICODE buffers in 16bit Unicode builds */ - Py_UNICODE high_val, low_val; - high_val = (Py_UNICODE) (0xD800 | (((character - 0x10000) >> 10) & ((1<<10)-1))); - low_val = (Py_UNICODE) (0xDC00 | ( (character - 0x10000) & ((1<<10)-1))); - for (pos=buffer; pos < buffer+length-1; pos++) { - if (unlikely(high_val == pos[0]) & unlikely(low_val == pos[1])) return 1; - } - return 0; - } - #endif - uchar = (Py_UNICODE) character; - for (pos=buffer; pos < buffer+length; pos++) { - if (unlikely(uchar == pos[0])) return 1; +#if PY_VERSION_HEX < 0x03090000 || (defined(PyUnicode_WCHAR_KIND) && defined(PyUnicode_AS_UNICODE)) +#if !defined(Py_UNICODE_SIZE) || Py_UNICODE_SIZE == 2 + if ((sizeof(Py_UNICODE) == 2) && unlikely(character > 65535)) { + return __Pyx_PyUnicodeBufferContainsUCS4_SP( + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), + character); + } else +#endif + { + return __Pyx_PyUnicodeBufferContainsUCS4_BMP( + __Pyx_PyUnicode_AS_UNICODE(unicode), + __Pyx_PyUnicode_GET_SIZE(unicode), + character); + } - return 0; +#endif } @@ -145,7 +235,7 @@ //@requires: BytesEquals static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else #if PY_MAJOR_VERSION < 3 @@ -185,6 +275,21 @@ if (length != __Pyx_PyUnicode_GET_LENGTH(s2)) { goto return_ne; } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + #if CYTHON_PEP393_ENABLED + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + #else + hash1 = ((PyUnicodeObject*)s1)->hash; + hash2 = ((PyUnicodeObject*)s2)->hash; + #endif + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif // len(s1) == len(s2) >= 1 (empty string is interned, and "s1 is not s2") kind = __Pyx_PyUnicode_KIND(s1); if (kind != __Pyx_PyUnicode_KIND(s2)) { @@ -210,6 +315,9 @@ } else { int result; PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + #if PY_MAJOR_VERSION < 3 + Py_XDECREF(owned_ref); + #endif if (!py_result) return -1; result = __Pyx_PyObject_IsTrue(py_result); @@ -238,7 +346,7 @@ //@requires: IncludeStringH static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else if (s1 == s2) { @@ -257,7 +365,16 @@ } else if (length == 1) { return (equals == Py_EQ); } else { - int result = memcmp(ps1, ps2, (size_t)length); + int result; +#if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); return (equals == Py_EQ) ? (result == 0) : (result != 0); } } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { @@ -294,7 +411,7 @@ if (wraparound | boundscheck) { length = PyByteArray_GET_SIZE(string); if (wraparound & unlikely(i < 0)) i += length; - if ((!boundscheck) || likely((0 <= i) & (i < length))) { + if ((!boundscheck) || likely(__Pyx_is_valid_index(i, length))) { return (unsigned char) (PyByteArray_AS_STRING(string)[i]); } else { PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); @@ -324,7 +441,7 @@ if (wraparound | boundscheck) { length = PyByteArray_GET_SIZE(string); if (wraparound & unlikely(i < 0)) i += length; - if ((!boundscheck) || likely((0 <= i) & (i < length))) { + if ((!boundscheck) || likely(__Pyx_is_valid_index(i, length))) { PyByteArray_AS_STRING(string)[i] = (char) v; return 0; } else { @@ -357,7 +474,7 @@ if (wraparound | boundscheck) { length = __Pyx_PyUnicode_GET_LENGTH(ustring); if (wraparound & unlikely(i < 0)) i += length; - if ((!boundscheck) || likely((0 <= i) & (i < length))) { + if ((!boundscheck) || likely(__Pyx_is_valid_index(i, length))) { return __Pyx_PyUnicode_READ_CHAR(ustring, i); } else { PyErr_SetString(PyExc_IndexError, "string index out of range"); @@ -369,6 +486,21 @@ } +/////////////// decode_c_string_utf16.proto /////////////// + +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 0; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16LE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = -1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} + /////////////// decode_cpp_string.proto /////////////// //@requires: IncludeCppStringH //@requires: decode_c_bytes @@ -390,6 +522,8 @@ /////////////// decode_c_string /////////////// //@requires: IncludeStringH +//@requires: decode_c_string_utf16 +//@substitute: naming /* duplicate code to avoid calling strlen() if start >= 0 and stop >= 0 */ static CYTHON_INLINE PyObject* __Pyx_decode_c_string( @@ -413,9 +547,9 @@ if (stop < 0) stop += length; } + if (unlikely(stop <= start)) + return __Pyx_NewRef($empty_unicode); length = stop - start; - if (unlikely(length <= 0)) - return PyUnicode_FromUnicode(NULL, 0); cstring += start; if (decode_func) { return decode_func(cstring, length, errors); @@ -432,6 +566,8 @@ PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); /////////////// decode_c_bytes /////////////// +//@requires: decode_c_string_utf16 +//@substitute: naming static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, @@ -448,9 +584,9 @@ } if (stop > length) stop = length; + if (unlikely(stop <= start)) + return __Pyx_NewRef($empty_unicode); length = stop - start; - if (unlikely(length <= 0)) - return PyUnicode_FromUnicode(NULL, 0); cstring += start; if (decode_func) { return decode_func(cstring, length, errors); @@ -489,6 +625,7 @@ PyObject* text, Py_ssize_t start, Py_ssize_t stop); /////////////// PyUnicode_Substring /////////////// +//@substitute: naming static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring( PyObject* text, Py_ssize_t start, Py_ssize_t stop) { @@ -504,9 +641,10 @@ stop += length; else if (stop > length) stop = length; - length = stop - start; - if (length <= 0) - return PyUnicode_FromUnicode(NULL, 0); + if (stop <= start) + return __Pyx_NewRef($empty_unicode); + if (start == 0 && stop == length) + return __Pyx_NewRef(text); #if CYTHON_PEP393_ENABLED return PyUnicode_FromKindAndData(PyUnicode_KIND(text), PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text), stop-start); @@ -533,9 +671,8 @@ /////////////// unicode_tailmatch.proto /////////////// -static int __Pyx_PyUnicode_Tailmatch(PyObject* s, PyObject* substr, - Py_ssize_t start, Py_ssize_t end, int direction); /*proto*/ - +static int __Pyx_PyUnicode_Tailmatch( + PyObject* s, PyObject* substr, Py_ssize_t start, Py_ssize_t end, int direction); /*proto*/ /////////////// unicode_tailmatch /////////////// @@ -543,26 +680,31 @@ // tuple of prefixes/suffixes, whereas it's much more common to // test for a single unicode string. -static int __Pyx_PyUnicode_Tailmatch(PyObject* s, PyObject* substr, - Py_ssize_t start, Py_ssize_t end, int direction) { - if (unlikely(PyTuple_Check(substr))) { - Py_ssize_t i, count = PyTuple_GET_SIZE(substr); - for (i = 0; i < count; i++) { - Py_ssize_t result; +static int __Pyx_PyUnicode_TailmatchTuple(PyObject* s, PyObject* substrings, + Py_ssize_t start, Py_ssize_t end, int direction) { + Py_ssize_t i, count = PyTuple_GET_SIZE(substrings); + for (i = 0; i < count; i++) { + Py_ssize_t result; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - result = PyUnicode_Tailmatch(s, PyTuple_GET_ITEM(substr, i), - start, end, direction); + result = PyUnicode_Tailmatch(s, PyTuple_GET_ITEM(substrings, i), + start, end, direction); #else - PyObject* sub = PySequence_ITEM(substr, i); - if (unlikely(!sub)) return -1; - result = PyUnicode_Tailmatch(s, sub, start, end, direction); - Py_DECREF(sub); + PyObject* sub = PySequence_ITEM(substrings, i); + if (unlikely(!sub)) return -1; + result = PyUnicode_Tailmatch(s, sub, start, end, direction); + Py_DECREF(sub); #endif - if (result) { - return (int) result; - } + if (result) { + return (int) result; } - return 0; + } + return 0; +} + +static int __Pyx_PyUnicode_Tailmatch(PyObject* s, PyObject* substr, + Py_ssize_t start, Py_ssize_t end, int direction) { + if (unlikely(PyTuple_Check(substr))) { + return __Pyx_PyUnicode_TailmatchTuple(s, substr, start, end, direction); } return (int) PyUnicode_Tailmatch(s, substr, start, end, direction); } @@ -633,26 +775,31 @@ return retval; } -static int __Pyx_PyBytes_Tailmatch(PyObject* self, PyObject* substr, - Py_ssize_t start, Py_ssize_t end, int direction) { - if (unlikely(PyTuple_Check(substr))) { - Py_ssize_t i, count = PyTuple_GET_SIZE(substr); - for (i = 0; i < count; i++) { - int result; +static int __Pyx_PyBytes_TailmatchTuple(PyObject* self, PyObject* substrings, + Py_ssize_t start, Py_ssize_t end, int direction) { + Py_ssize_t i, count = PyTuple_GET_SIZE(substrings); + for (i = 0; i < count; i++) { + int result; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - result = __Pyx_PyBytes_SingleTailmatch(self, PyTuple_GET_ITEM(substr, i), - start, end, direction); + result = __Pyx_PyBytes_SingleTailmatch(self, PyTuple_GET_ITEM(substrings, i), + start, end, direction); #else - PyObject* sub = PySequence_ITEM(substr, i); - if (unlikely(!sub)) return -1; - result = __Pyx_PyBytes_SingleTailmatch(self, sub, start, end, direction); - Py_DECREF(sub); + PyObject* sub = PySequence_ITEM(substrings, i); + if (unlikely(!sub)) return -1; + result = __Pyx_PyBytes_SingleTailmatch(self, sub, start, end, direction); + Py_DECREF(sub); #endif - if (result) { - return result; - } + if (result) { + return result; } - return 0; + } + return 0; +} + +static int __Pyx_PyBytes_Tailmatch(PyObject* self, PyObject* substr, + Py_ssize_t start, Py_ssize_t end, int direction) { + if (unlikely(PyTuple_Check(substr))) { + return __Pyx_PyBytes_TailmatchTuple(self, substr, start, end, direction); } return __Pyx_PyBytes_SingleTailmatch(self, substr, start, end, direction); @@ -689,15 +836,15 @@ /////////////// bytes_index /////////////// static CYTHON_INLINE char __Pyx_PyBytes_GetItemInt(PyObject* bytes, Py_ssize_t index, int check_bounds) { + if (index < 0) + index += PyBytes_GET_SIZE(bytes); if (check_bounds) { Py_ssize_t size = PyBytes_GET_SIZE(bytes); - if (unlikely(index >= size) | ((index < 0) & unlikely(index < -size))) { + if (unlikely(!__Pyx_is_valid_index(index, size))) { PyErr_SetString(PyExc_IndexError, "string index out of range"); - return -1; + return (char) -1; } } - if (index < 0) - index += PyBytes_GET_SIZE(bytes); return PyBytes_AS_STRING(bytes)[index]; } @@ -742,25 +889,29 @@ //@substitute: naming static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, - CYTHON_UNUSED Py_UCS4 max_char) { + Py_UCS4 max_char) { #if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *result_uval; - int result_ukind; + int result_ukind, kind_shift; Py_ssize_t i, char_pos; void *result_udata; + CYTHON_MAYBE_UNUSED_VAR(max_char); #if CYTHON_PEP393_ENABLED // Py 3.3+ (post PEP-393) result_uval = PyUnicode_New(result_ulength, max_char); if (unlikely(!result_uval)) return NULL; result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; result_udata = PyUnicode_DATA(result_uval); #else // Py 2.x/3.2 (pre PEP-393) result_uval = PyUnicode_FromUnicode(NULL, result_ulength); if (unlikely(!result_uval)) return NULL; result_ukind = sizeof(Py_UNICODE); + kind_shift = (result_ukind == 4) ? 2 : result_ukind - 1; result_udata = PyUnicode_AS_UNICODE(result_uval); #endif + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); char_pos = 0; for (i=0; i < value_count; i++) { @@ -773,14 +924,14 @@ ulength = __Pyx_PyUnicode_GET_LENGTH(uval); if (unlikely(!ulength)) continue; - if (unlikely(char_pos + ulength < 0)) + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) goto overflow; ukind = __Pyx_PyUnicode_KIND(uval); udata = __Pyx_PyUnicode_DATA(uval); if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, ulength * result_ukind); + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); } else { - #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); #else Py_ssize_t j; @@ -800,8 +951,9 @@ return NULL; #else // non-CPython fallback - result_ulength++; - value_count++; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + CYTHON_UNUSED_VAR(value_count); return PyUnicode_Join($empty_unicode, value_tuple); #endif } @@ -852,8 +1004,8 @@ #else // non-CPython { - uval = NULL; PyObject *sign = NULL, *padding = NULL; + uval = NULL; if (uoffset > 0) { prepend_sign = !!prepend_sign; if (uoffset > prepend_sign) { @@ -927,7 +1079,7 @@ { // CPython calls PyNumber_Index() internally ival = __Pyx_PyIndex_AsSsize_t(value); - if (unlikely((ival < 0) | (ival > 255))) { + if (unlikely(!__Pyx_is_valid_index(ival, 256))) { if (ival == -1 && PyErr_Occurred()) return -1; goto bad_range; @@ -949,7 +1101,7 @@ static CYTHON_INLINE int __Pyx_PyByteArray_Append(PyObject* bytearray, int value) { PyObject *pyval, *retval; #if CYTHON_COMPILING_IN_CPYTHON - if (likely((value >= 0) & (value <= 255))) { + if (likely(__Pyx_is_valid_index(value, 256))) { Py_ssize_t n = Py_SIZE(bytearray); if (likely(n != PY_SSIZE_T_MAX)) { if (unlikely(PyByteArray_Resize(bytearray, n + 1) < 0)) @@ -1037,11 +1189,12 @@ likely(PyString_CheckExact(s)) ? PyUnicode_FromEncodedObject(s, NULL, "strict") : \ PyObject_Format(s, f)) #elif CYTHON_USE_TYPE_SLOTS - // Py3 nicely returns unicode strings from str() which makes this quite efficient for builtin types + // Py3 nicely returns unicode strings from str() and repr(), which makes this quite efficient for builtin types. + // In Py3.8+, tp_str() delegates to tp_repr(), so we call tp_repr() directly here. #define __Pyx_PyObject_FormatSimple(s, f) ( \ likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) : \ - likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_str(s) : \ - likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_str(s) : \ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) : \ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) : \ PyObject_Format(s, f)) #else #define __Pyx_PyObject_FormatSimple(s, f) ( \ @@ -1076,3 +1229,46 @@ Py_DECREF(s); return result; } + + +//////////////////// PyUnicode_Unicode.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj);/*proto*/ + +//////////////////// PyUnicode_Unicode //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = PYUNICODE("None"); + return __Pyx_NewRef(obj); +} + + +//////////////////// PyObject_Unicode.proto //////////////////// + +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyObject_Unicode(obj) \ + (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj)) +#else +#define __Pyx_PyObject_Unicode(obj) \ + (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Unicode(obj)) +#endif + + +//////////////////// PyStr_Str.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj);/*proto*/ + +//////////////////// PyStr_Str //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = PYIDENT("None"); + return __Pyx_NewRef(obj); +} + + +//////////////////// PyObject_Str.proto //////////////////// + +#define __Pyx_PyObject_Str(obj) \ + (likely(PyString_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj)) diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/TestCythonScope.pyx cython-0.20.1+1~202203241016-9537/Cython/Utility/TestCythonScope.pyx --- cython-0.20.1+1~201611251650-6686/Cython/Utility/TestCythonScope.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/TestCythonScope.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,11 @@ ########## TestClass ########## # These utilities are for testing purposes +# The "cythonscope" test calls METH_O functions with their (self, arg) signature. +# cython: always_allow_keywords=False + +from __future__ import print_function + cdef extern from *: cdef object __pyx_test_dep(object) @@ -12,32 +17,32 @@ self.value = value def __str__(self): - return 'TestClass(%d)' % self.value + return f'TestClass({self.value})' cdef cdef_method(self, int value): - print 'Hello from cdef_method', value + print('Hello from cdef_method', value) cpdef cpdef_method(self, int value): - print 'Hello from cpdef_method', value + print('Hello from cpdef_method', value) def def_method(self, int value): - print 'Hello from def_method', value + print('Hello from def_method', value) @cname('cdef_cname') cdef cdef_cname_method(self, int value): - print "Hello from cdef_cname_method", value + print("Hello from cdef_cname_method", value) @cname('cpdef_cname') cpdef cpdef_cname_method(self, int value): - print "Hello from cpdef_cname_method", value + print("Hello from cpdef_cname_method", value) @cname('def_cname') def def_cname_method(self, int value): - print "Hello from def_cname_method", value + print("Hello from def_cname_method", value) @cname('__pyx_test_call_other_cy_util') cdef test_call(obj): - print 'test_call' + print('test_call') __pyx_test_dep(obj) @cname('__pyx_TestClass_New') @@ -46,19 +51,20 @@ ########### TestDep ########## +from __future__ import print_function + @cname('__pyx_test_dep') cdef test_dep(obj): - print 'test_dep', obj + print('test_dep', obj) ########## TestScope ########## @cname('__pyx_testscope') cdef object _testscope(int value): - return "hello from cython scope, value=%d" % value + return f"hello from cython scope, value={value}" ########## View.TestScope ########## @cname('__pyx_view_testscope') cdef object _testscope(int value): - return "hello from cython.view scope, value=%d" % value - + return f"hello from cython.view scope, value={value}" diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utility/TypeConversion.c cython-0.20.1+1~202203241016-9537/Cython/Utility/TypeConversion.c --- cython-0.20.1+1~201611251650-6686/Cython/Utility/TypeConversion.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utility/TypeConversion.c 2022-03-24 10:16:46.000000000 +0000 @@ -16,6 +16,14 @@ (is_signed || likely(v < (type)PY_SSIZE_T_MAX || \ v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + // Optimisation from Section 14.2 "Bounds Checking" in + // https://www.agner.org/optimize/optimizing_cpp.pdf + // See https://bugs.python.org/issue28397 + // The cast to unsigned effectively tests for "0 <= i < limit". + return (size_t) i < (size_t) limit; +} + // fast and unsafe abs(Py_ssize_t) that ignores the overflow for (-PY_SSIZE_T_MAX-1) #if defined (__cplusplus) && __cplusplus >= 201103L #include @@ -24,10 +32,10 @@ #define __Pyx_sst_abs(value) abs(value) #elif SIZEOF_LONG >= SIZEOF_SIZE_T #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) +#elif defined (_MSC_VER) // abs() is defined for long, but 64-bits type on MSVC is long long. // Use MS-specific _abs64 instead. - #define __Pyx_sst_abs(value) _abs64(value) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define __Pyx_sst_abs(value) llabs(value) #elif defined (__GNUC__) @@ -37,8 +45,8 @@ #define __Pyx_sst_abs(value) ((value<0) ? -value : value) #endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) #define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) @@ -54,23 +62,38 @@ #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) #define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) #define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) #define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 +// There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3. +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u) +{ + const wchar_t *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen #endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) @@ -79,12 +102,17 @@ #define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) #define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +#define __Pyx_PySequence_Tuple(obj) \ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) + static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); #if CYTHON_ASSUME_SAFE_MACROS #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) @@ -98,7 +126,8 @@ #else #define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) #endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +// __Pyx_PyNumber_Float is now in it's own section since it has dependencies (needed to make +// string conversion work the same in all circumstances) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; @@ -169,7 +198,7 @@ if (!default_encoding) goto bad; default_encoding_c = PyBytes_AsString(default_encoding); if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c) + 1); if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); Py_DECREF(default_encoding); @@ -189,59 +218,71 @@ return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); } -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { +// Py3.7 returns a "const char*" for unicode strings +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - // borrowed reference, cached internally in 'o' by CPython - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +#if !CYTHON_PEP393_ENABLED +static const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + char* defenc_c; + // borrowed reference, cached internally in 'o' by CPython + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - // raise the error - PyUnicode_AsASCIIString(o); - return NULL; - } + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + // raise the error + PyUnicode_AsASCIIString(o); + return NULL; } } + } #endif /*__PYX_DEFAULT_STRING_ENCODING_IS_ASCII*/ - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; -#else /* PY_VERSION_HEX < 0x03030000 */ - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +} + +#else /* CYTHON_PEP393_ENABLED: */ + +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - // cached for the lifetime of the object - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - // raise the error - PyUnicode_AsASCIIString(o); - return NULL; - } + if (likely(PyUnicode_IS_ASCII(o))) { + // cached for the lifetime of the object + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + // raise the error + PyUnicode_AsASCIIString(o); + return NULL; + } #else /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ - return PyUnicode_AsUTF8AndSize(o, length); + return PyUnicode_AsUTF8AndSize(o, length); #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII */ -#endif /* PY_VERSION_HEX < 0x03030000 */ +} +#endif /* CYTHON_PEP393_ENABLED */ +#endif + +// Py3.7 returns a "const char*" for unicode strings +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) +#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); @@ -265,6 +306,40 @@ else return PyObject_IsTrue(x); } +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} + +static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetName(Py_TYPE(result)); +#if PY_MAJOR_VERSION >= 3 + if (PyLong_Check(result)) { + // CPython issue #17576: warn if 'result' not of exact type int. + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type " __Pyx_FMT_TYPENAME ")", + type_name, type_name, result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} + static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { #if CYTHON_USE_TYPE_SLOTS PyNumberMethods *m; @@ -272,9 +347,9 @@ const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) + if (likely(PyInt_Check(x) || PyLong_Check(x))) #else - if (PyLong_Check(x)) + if (likely(PyLong_Check(x))) #endif return __Pyx_NewRef(x); #if CYTHON_USE_TYPE_SLOTS @@ -282,32 +357,30 @@ #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; - res = PyNumber_Int(x); + res = m->nb_int(x); } else if (m && m->nb_long) { name = "long"; - res = PyNumber_Long(x); + res = m->nb_long(x); } #else - if (m && m->nb_int) { + if (likely(m && m->nb_int)) { name = "int"; - res = PyNumber_Long(x); + res = m->nb_int(x); } #endif #else - res = PyNumber_Int(x); + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Int(x); + } #endif - if (res) { + if (likely(res)) { #if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { + if (unlikely(!PyInt_Check(res) && !PyLong_Check(res))) { #else - if (!PyLong_Check(res)) { + if (unlikely(!PyLong_CheckExact(res))) { #endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; + return __Pyx_PyNumber_IntOrLongWrongResultType(res, name); } } else if (!PyErr_Occurred()) { @@ -327,7 +400,7 @@ if (sizeof(Py_ssize_t) >= sizeof(long)) return PyInt_AS_LONG(b); else - return PyInt_AsSsize_t(x); + return PyInt_AsSsize_t(b); } #endif if (likely(PyLong_CheckExact(b))) { @@ -362,10 +435,92 @@ return ival; } + +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); +#if PY_MAJOR_VERSION < 3 + } else if (likely(PyInt_CheckExact(o))) { + return PyInt_AS_LONG(o); +#endif + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyInt_AsLong(x); + Py_DECREF(x); + return ival; + } +} + + +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} + + static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } +/////////////// pynumber_float.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj); /* proto */ +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : __Pyx__PyNumber_Float(x)) + +/////////////// pynumber_float /////////////// +//@requires: Optimize.c::pybytes_as_double +//@requires: Optimize.c::pyunicode_as_double + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) { + // 'obj is PyFloat' is handled in the calling macro + double val; + if (PyLong_CheckExact(obj)) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)obj)->ob_digit; + switch (Py_SIZE(obj)) { + case 0: + val = 0.0; + goto no_error; + // single digit PyLong values always cast safely to double + case 1: + val = (double) digits[0]; + goto no_error; + case -1: + val = (double) - (sdigit) digits[0]; + goto no_error; + default: + val = PyLong_AsDouble(obj); + } +#else + val = PyLong_AsDouble(obj); +#endif + } else if (PyUnicode_CheckExact(obj)) { + val = __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + val = __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + val = __Pyx_PyByteArray_AsDouble(obj); + } else { + return PyNumber_Float(obj); + } + + if (unlikely(val == -1 && PyErr_Occurred())) { + return NULL; + } +no_error: + return PyFloat_FromDouble(val); +} + +/////////////// GCCDiagnostics.proto /////////////// + +// GCC diagnostic pragmas were introduced in GCC 4.6 +// Used to silence conversion warnings that are ok but cannot be avoided. +#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + /////////////// ToPyCTupleUtility.proto /////////////// static PyObject* {{funcname}}({{struct_type_decl}}); @@ -399,7 +554,10 @@ {{struct_type_decl}} result; if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != {{size}}) { - PyErr_Format(PyExc_TypeError, "Expected %.16s of size %d, got %.200s", "a tuple", {{size}}, Py_TYPE(o)->tp_name); + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); + PyErr_Format(PyExc_TypeError, + "Expected a tuple of size %d, got " __Pyx_FMT_TYPENAME, {{size}}, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } @@ -474,18 +632,23 @@ /////////////// ObjectAsUCS4 /////////////// -static Py_UCS4 __Pyx__PyObject_AsPy_UCS4(PyObject* x) { - long ival; - ival = __Pyx_PyInt_As_long(x); - if (unlikely(ival < 0)) { +static Py_UCS4 __Pyx__PyObject_AsPy_UCS4_raise_error(long ival) { + if (ival < 0) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_OverflowError, "cannot convert negative value to Py_UCS4"); - return (Py_UCS4)-1; - } else if (unlikely(ival > 1114111)) { + } else { PyErr_SetString(PyExc_OverflowError, "value too large to convert to Py_UCS4"); - return (Py_UCS4)-1; + } + return (Py_UCS4)-1; +} + +static Py_UCS4 __Pyx__PyObject_AsPy_UCS4(PyObject* x) { + long ival; + ival = __Pyx_PyInt_As_long(x); + if (unlikely(!__Pyx_is_valid_index(ival, 1114111 + 1))) { + return __Pyx__PyObject_AsPy_UCS4_raise_error(ival); } return (Py_UCS4)ival; } @@ -527,14 +690,16 @@ #endif ival = __Pyx_PyInt_As_long(x); } - if (unlikely(ival < 0)) { - if (!PyErr_Occurred()) + if (unlikely(!__Pyx_is_valid_index(ival, maxval + 1))) { + if (ival < 0) { + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_OverflowError, + "cannot convert negative value to Py_UNICODE"); + return (Py_UNICODE)-1; + } else { PyErr_SetString(PyExc_OverflowError, - "cannot convert negative value to Py_UNICODE"); - return (Py_UNICODE)-1; - } else if (unlikely(ival > maxval)) { - PyErr_SetString(PyExc_OverflowError, - "value too large to convert to Py_UNICODE"); + "value too large to convert to Py_UNICODE"); + } return (Py_UNICODE)-1; } return (Py_UNICODE)ival; @@ -546,9 +711,17 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value); /////////////// CIntToPy /////////////// +//@requires: GCCDiagnostics static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif const {{TYPE}} neg_one = ({{TYPE}}) -1, const_zero = ({{TYPE}}) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif const int is_unsigned = neg_one > const_zero; if (is_unsigned) { if (sizeof({{TYPE}}) < sizeof(long)) { @@ -605,7 +778,8 @@ }; static const char DIGITS_HEX[2*16+1] = { - "0123456789abcdef0123456789ABCDEF" + "0123456789abcdef" + "0123456789ABCDEF" }; @@ -614,10 +788,10 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t width, char padding_char, char format_char); /////////////// CIntToPyUnicode /////////////// +//@requires: StringTools.c::IncludeStringH //@requires: StringTools.c::BuildPyUnicode //@requires: CIntToDigits - -#include +//@requires: GCCDiagnostics // NOTE: inlining because most arguments are constant, which collapses lots of code below @@ -627,53 +801,59 @@ // 'dpos' points to end of digits array + 1 initially to allow for pre-decrement looping char *dpos, *end = digits + sizeof({{TYPE}})*3+2; const char *hex_digits = DIGITS_HEX; - Py_ssize_t ulength; - int length, prepend_sign, last_one_off; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; {{TYPE}} remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif const {{TYPE}} neg_one = ({{TYPE}}) -1, const_zero = ({{TYPE}}) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif const int is_unsigned = neg_one > const_zero; if (format_char == 'X') { hex_digits += 16; format_char = 'x'; - }; + } // surprise: even trivial sprintf() calls don't get optimised in gcc (4.8) remaining = value; /* not using abs(value) to avoid overflow problems */ last_one_off = 0; dpos = end; - while (remaining != 0) { + do { int digit_pos; switch (format_char) { case 'o': - digit_pos = abs(remaining % (8*8)); - remaining = remaining / (8*8); + digit_pos = abs((int)(remaining % (8*8))); + remaining = ({{TYPE}}) (remaining / (8*8)); dpos -= 2; - *(uint16_t*)dpos = ((uint16_t*)DIGIT_PAIRS_8)[digit_pos]; /* copy 2 digits at a time */ + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); /* copy 2 digits at a time, unaligned */ last_one_off = (digit_pos < 8); break; case 'd': - digit_pos = abs(remaining % (10*10)); - remaining = remaining / (10*10); + digit_pos = abs((int)(remaining % (10*10))); + remaining = ({{TYPE}}) (remaining / (10*10)); dpos -= 2; - *(uint16_t*)dpos = ((uint16_t*)DIGIT_PAIRS_10)[digit_pos]; /* copy 2 digits at a time */ + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); /* copy 2 digits at a time, unaligned */ last_one_off = (digit_pos < 10); break; case 'x': - *(--dpos) = hex_digits[abs(remaining % 16)]; - remaining = remaining / 16; + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = ({{TYPE}}) (remaining / 16); break; default: assert(0); break; } - } - if (last_one_off) { - assert(*dpos == '0'); - dpos++; - } else if (unlikely(dpos == end)) { - *(--dpos) = '0'; - } + } while (unlikely(remaining != 0)); + + // Correct dpos by 1 if we read an excess digit. + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; ulength = length; prepend_sign = 0; @@ -693,7 +873,7 @@ if (ulength == 1) { return PyUnicode_FromOrdinal(*dpos); } - return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, length, prepend_sign, padding_char); + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); } @@ -756,15 +936,23 @@ /////////////// CIntFromPy /////////////// //@requires: CIntFromPyVerify +//@requires: GCCDiagnostics {{py: from Cython.Utility import pylong_join }} static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif const {{TYPE}} neg_one = ({{TYPE}}) -1, const_zero = ({{TYPE}}) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof({{TYPE}}) < sizeof(long)) { + if ((sizeof({{TYPE}}) < sizeof(long))) { __PYX_VERIFY_RETURN_INT({{TYPE}}, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); @@ -784,10 +972,10 @@ case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0]) {{for _size in (2, 3, 4)}} case {{_size}}: - if (8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; } } @@ -809,10 +997,10 @@ goto raise_neg_overflow; } #endif - if (sizeof({{TYPE}}) <= sizeof(unsigned long)) { + if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } @@ -827,10 +1015,10 @@ {{for _size in (2, 3, 4)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); } } @@ -839,18 +1027,18 @@ {{endfor}} } #endif - if (sizeof({{TYPE}}) <= sizeof(long)) { + if ((sizeof({{TYPE}}) <= sizeof(long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) +#if (CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); + "_PyLong_AsByteArray() not available, cannot convert large numbers"); #else {{TYPE}} val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utils.pxd cython-0.20.1+1~202203241016-9537/Cython/Utils.pxd --- cython-0.20.1+1~201611251650-6686/Cython/Utils.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utils.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,3 @@ + +cdef class _TryFinallyGeneratorContextManager: + cdef object _gen diff -Nru cython-0.20.1+1~201611251650-6686/Cython/Utils.py cython-0.20.1+1~202203241016-9537/Cython/Utils.py --- cython-0.20.1+1~201611251650-6686/Cython/Utils.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Cython/Utils.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,39 +1,124 @@ -# -# Cython -- Things that don't belong -# anywhere else in particular -# +""" +Cython -- Things that don't belong anywhere else in particular +""" from __future__ import absolute_import +import cython + +cython.declare( + basestring=object, + os=object, sys=object, re=object, io=object, codecs=object, glob=object, shutil=object, tempfile=object, + cython_version=object, + _function_caches=list, _parse_file_version=object, _match_file_encoding=object, +) + try: from __builtin__ import basestring except ImportError: basestring = str +try: + FileNotFoundError +except NameError: + FileNotFoundError = OSError + import os import sys import re import io import codecs +import glob import shutil -from contextlib import contextmanager +import tempfile +from functools import wraps + +from . import __version__ as cython_version + +PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd") + +_build_cache_name = "__{0}_cache".format +_CACHE_NAME_PATTERN = re.compile(r"^__(.+)_cache$") modification_time = os.path.getmtime +GENERATED_BY_MARKER = "/* Generated by Cython %s */" % cython_version +GENERATED_BY_MARKER_BYTES = GENERATED_BY_MARKER.encode('us-ascii') + + +class _TryFinallyGeneratorContextManager(object): + """ + Fast, bare minimum @contextmanager, only for try-finally, not for exception handling. + """ + def __init__(self, gen): + self._gen = gen + + def __enter__(self): + return next(self._gen) + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + next(self._gen) + except (StopIteration, GeneratorExit): + pass + + +def try_finally_contextmanager(gen_func): + @wraps(gen_func) + def make_gen(*args, **kwargs): + return _TryFinallyGeneratorContextManager(gen_func(*args, **kwargs)) + return make_gen + + +_function_caches = [] + + +def clear_function_caches(): + for cache in _function_caches: + cache.clear() + def cached_function(f): cache = {} + _function_caches.append(cache) uncomputed = object() + + @wraps(f) def wrapper(*args): res = cache.get(args, uncomputed) if res is uncomputed: res = cache[args] = f(*args) return res + wrapper.uncached = f return wrapper + +def _find_cache_attributes(obj): + """The function iterates over the attributes of the object and, + if it finds the name of the cache, it returns it and the corresponding method name. + The method may not be present in the object. + """ + for attr_name in dir(obj): + match = _CACHE_NAME_PATTERN.match(attr_name) + if match is not None: + yield attr_name, match.group(1) + + +def clear_method_caches(obj): + """Removes every cache found in the object, + if a corresponding method exists for that cache. + """ + for cache_name, method_name in _find_cache_attributes(obj): + if hasattr(obj, method_name): + delattr(obj, cache_name) + # if there is no corresponding method, then we assume + # that this attribute was not created by our cached method + + def cached_method(f): - cache_name = '__%s_cache' % f.__name__ + cache_name = _build_cache_name(f.__name__) + def wrapper(self, *args): cache = getattr(self, cache_name, None) if cache is None: @@ -43,8 +128,10 @@ return cache[args] res = cache[args] = f(self, *args) return res + return wrapper + def replace_suffix(path, newsuf): base, _ = os.path.splitext(path) return base + newsuf @@ -70,6 +157,9 @@ # failed compilation. # Also sets access and modification times back to # those specified by st (a stat struct). + if not is_cython_generated_file(path, allow_failed=True, if_not_found=False): + return + try: f = open_new_file(path) except EnvironmentError: @@ -81,6 +171,42 @@ if st: os.utime(path, (st.st_atime, st.st_mtime-1)) + +def is_cython_generated_file(path, allow_failed=False, if_not_found=True): + failure_marker = b"#error Do not use this file, it is the result of a failed Cython compilation." + file_content = None + if os.path.exists(path): + try: + with open(path, "rb") as f: + file_content = f.read(len(failure_marker)) + except (OSError, IOError): + pass # Probably just doesn't exist any more + + if file_content is None: + # file does not exist (yet) + return if_not_found + + return ( + # Cython C file? + file_content.startswith(b"/* Generated by Cython ") or + # Cython output file after previous failures? + (allow_failed and file_content == failure_marker) or + # Let's allow overwriting empty files as well. They might have resulted from previous failures. + not file_content + ) + + +def file_generated_by_this_cython(path): + file_content = b'' + if os.path.exists(path): + try: + with open(path, "rb") as f: + file_content = f.read(len(GENERATED_BY_MARKER_BYTES)) + except (OSError, IOError): + pass # Probably just doesn't exist any more + return file_content and file_content.startswith(GENERATED_BY_MARKER_BYTES) + + def file_newer_than(path, time): ftime = modification_time(path) return ftime > time @@ -114,54 +240,6 @@ @cached_function -def search_include_directories(dirs, qualified_name, suffix, pos, - include=False, sys_path=False): - # Search the list of include directories for the given - # file name. If a source file position is given, first - # searches the directory containing that file. Returns - # None if not found, but does not report an error. - # The 'include' option will disable package dereferencing. - # If 'sys_path' is True, also search sys.path. - if sys_path: - dirs = dirs + tuple(sys.path) - if pos: - file_desc = pos[0] - from Cython.Compiler.Scanning import FileSourceDescriptor - if not isinstance(file_desc, FileSourceDescriptor): - raise RuntimeError("Only file sources for code supported") - if include: - dirs = (os.path.dirname(file_desc.filename),) + dirs - else: - dirs = (find_root_package_dir(file_desc.filename),) + dirs - - dotted_filename = qualified_name - if suffix: - dotted_filename += suffix - if not include: - names = qualified_name.split('.') - package_names = tuple(names[:-1]) - module_name = names[-1] - module_filename = module_name + suffix - package_filename = "__init__" + suffix - - for dir in dirs: - path = os.path.join(dir, dotted_filename) - if path_exists(path): - return path - if not include: - package_dir = check_package_dir(dir, package_names) - if package_dir is not None: - path = os.path.join(package_dir, module_filename) - if path_exists(path): - return path - path = os.path.join(dir, package_dir, module_name, - package_filename) - if path_exists(path): - return path - return None - - -@cached_function def find_root_package_dir(file_path): dir = os.path.dirname(file_path) if file_path == dir: @@ -171,24 +249,31 @@ else: return dir + @cached_function -def check_package_dir(dir, package_names): +def check_package_dir(dir_path, package_names): + namespace = True for dirname in package_names: - dir = os.path.join(dir, dirname) - if not is_package_dir(dir): - return None - return dir + dir_path = os.path.join(dir_path, dirname) + has_init = contains_init(dir_path) + if has_init: + namespace = False + return dir_path, namespace + @cached_function -def is_package_dir(dir_path): - for filename in ("__init__.py", - "__init__.pyc", - "__init__.pyx", - "__init__.pxd"): +def contains_init(dir_path): + for filename in PACKAGE_FILES: path = os.path.join(dir_path, filename) if path_exists(path): return 1 + +def is_package_dir(dir_path): + if contains_init(dir_path): + return 1 + + @cached_function def path_exists(path): # try on the filesystem first @@ -213,6 +298,40 @@ pass return False + +_parse_file_version = re.compile(r".*[.]cython-([0-9]+)[.][^./\\]+$").findall + + +@cached_function +def find_versioned_file(directory, filename, suffix, + _current_version=int(re.sub(r"^([0-9]+)[.]([0-9]+).*", r"\1\2", cython_version))): + """ + Search a directory for versioned pxd files, e.g. "lib.cython-30.pxd" for a Cython 3.0+ version. + + @param directory: the directory to search + @param filename: the filename without suffix + @param suffix: the filename extension including the dot, e.g. ".pxd" + @return: the file path if found, or None + """ + assert not suffix or suffix[:1] == '.' + path_prefix = os.path.join(directory, filename) + + matching_files = glob.glob(path_prefix + ".cython-*" + suffix) + path = path_prefix + suffix + if not os.path.exists(path): + path = None + best_match = (-1, path) # last resort, if we do not have versioned .pxd files + + for path in matching_files: + versions = _parse_file_version(path) + if versions: + int_version = int(versions[0]) + # Let's assume no duplicates. + if best_match[0] < int_version <= _current_version: + best_match = (int_version, path) + return best_match[1] + + # file name encodings def decode_filename(filename): @@ -226,46 +345,33 @@ pass return filename -# support for source file encoding detection - -_match_file_encoding = re.compile(u"coding[:=]\s*([-\w.]+)").search +# support for source file encoding detection -def detect_file_encoding(source_filename): - f = open_source_file(source_filename, encoding="UTF-8", error_handling='ignore') - try: - return detect_opened_file_encoding(f) - finally: - f.close() +_match_file_encoding = re.compile(br"(\w*coding)[:=]\s*([-\w.]+)").search -def detect_opened_file_encoding(f): +def detect_opened_file_encoding(f, default='UTF-8'): # PEPs 263 and 3120 - # Most of the time the first two lines fall in the first 250 chars, + # Most of the time the first two lines fall in the first couple of hundred chars, # and this bulk read/split is much faster. - lines = f.read(250).split(u"\n") - if len(lines) > 1: - m = _match_file_encoding(lines[0]) + lines = () + start = b'' + while len(lines) < 3: + data = f.read(500) + start += data + lines = start.split(b"\n") + if not data: + break + + m = _match_file_encoding(lines[0]) + if m and m.group(1) != b'c_string_encoding': + return m.group(2).decode('iso8859-1') + elif len(lines) > 1: + m = _match_file_encoding(lines[1]) if m: - return m.group(1) - elif len(lines) > 2: - m = _match_file_encoding(lines[1]) - if m: - return m.group(1) - else: - return "UTF-8" - # Fallback to one-char-at-a-time detection. - f.seek(0) - chars = [] - for i in range(2): - c = f.read(1) - while c and c != u'\n': - chars.append(c) - c = f.read(1) - encoding = _match_file_encoding(u''.join(chars)) - if encoding: - return encoding.group(1) - return "UTF-8" + return m.group(2).decode('iso8859-1') + return default def skip_bom(f): @@ -278,32 +384,33 @@ f.seek(0) -def open_source_file(source_filename, mode="r", - encoding=None, error_handling=None): - if encoding is None: - # Most of the time the coding is unspecified, so be optimistic that - # it's UTF-8. - f = open_source_file(source_filename, encoding="UTF-8", mode=mode, error_handling='ignore') - encoding = detect_opened_file_encoding(f) - if encoding == "UTF-8" and error_handling == 'ignore': +def open_source_file(source_filename, encoding=None, error_handling=None): + stream = None + try: + if encoding is None: + # Most of the time the encoding is not specified, so try hard to open the file only once. + f = io.open(source_filename, 'rb') + encoding = detect_opened_file_encoding(f) f.seek(0) - skip_bom(f) - return f + stream = io.TextIOWrapper(f, encoding=encoding, errors=error_handling) else: - f.close() + stream = io.open(source_filename, encoding=encoding, errors=error_handling) - if not os.path.exists(source_filename): + except OSError: + if os.path.exists(source_filename): + raise # File is there, but something went wrong reading from it. + # Allow source files to be in zip files etc. try: loader = __loader__ if source_filename.startswith(loader.archive): - return open_source_from_loader( + stream = open_source_from_loader( loader, source_filename, encoding, error_handling) except (NameError, AttributeError): pass - stream = io.open(source_filename, mode=mode, - encoding=encoding, errors=error_handling) + if stream is None: + raise FileNotFoundError(source_filename) skip_bom(stream) return stream @@ -355,7 +462,8 @@ @cached_function def get_cython_cache_dir(): - """get the cython cache dir + r""" + Return the base directory containing Cython's caches. Priority: @@ -383,48 +491,64 @@ return os.path.expanduser(os.path.join('~', '.cython')) -@contextmanager +@try_finally_contextmanager def captured_fd(stream=2, encoding=None): - pipe_in = t = None orig_stream = os.dup(stream) # keep copy of original stream try: - pipe_in, pipe_out = os.pipe() - os.dup2(pipe_out, stream) # replace stream by copy of pipe - try: - os.close(pipe_out) # close original pipe-out stream - data = [] - - def copy(): - try: - while True: - d = os.read(pipe_in, 1000) - if d: - data.append(d) - else: - break - finally: - os.close(pipe_in) + with tempfile.TemporaryFile(mode="a+b") as temp_file: + def read_output(_output=[b'']): + if not temp_file.closed: + temp_file.seek(0) + _output[0] = temp_file.read() + return _output[0] + os.dup2(temp_file.fileno(), stream) # replace stream by copy of pipe def get_output(): - output = b''.join(data) - if encoding: - output = output.decode(encoding) - return output - - from threading import Thread - t = Thread(target=copy) - t.daemon = True # just in case - t.start() + result = read_output() + return result.decode(encoding) if encoding else result + yield get_output - finally: + # note: @contextlib.contextmanager requires try-finally here os.dup2(orig_stream, stream) # restore original stream - if t is not None: - t.join() + read_output() # keep the output in case it's used after closing the context manager finally: os.close(orig_stream) -def print_bytes(s, end=b'\n', file=sys.stdout, flush=True): +def get_encoding_candidates(): + candidates = [sys.getdefaultencoding()] + for stream in (sys.stdout, sys.stdin, sys.__stdout__, sys.__stdin__): + encoding = getattr(stream, 'encoding', None) + # encoding might be None (e.g. somebody redirects stdout): + if encoding is not None and encoding not in candidates: + candidates.append(encoding) + return candidates + + +def prepare_captured(captured): + captured_bytes = captured.strip() + if not captured_bytes: + return None + for encoding in get_encoding_candidates(): + try: + return captured_bytes.decode(encoding) + except UnicodeDecodeError: + pass + # last resort: print at least the readable ascii parts correctly. + return captured_bytes.decode('latin-1') + + +def print_captured(captured, output, header_line=None): + captured = prepare_captured(captured) + if captured: + if header_line: + output.write(header_line) + output.write(captured) + + +def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): + if header_text: + file.write(header_text) # note: text! => file.write() instead of out.write() file.flush() try: out = file.buffer # Py3 @@ -436,17 +560,24 @@ if flush: out.flush() -class LazyStr: - def __init__(self, callback): - self.callback = callback - def __str__(self): - return self.callback() - def __repr__(self): - return self.callback() - def __add__(self, right): - return self.callback() + right - def __radd__(self, left): - return left + self.callback() + +class OrderedSet(object): + def __init__(self, elements=()): + self._list = [] + self._set = set() + self.update(elements) + + def __iter__(self): + return iter(self._list) + + def update(self, elements): + for e in elements: + self.add(e) + + def add(self, e): + if e not in self._set: + self._list.append(e) + self._set.add(e) # Class decorator that adds a metaclass and recreates the class with it. @@ -465,3 +596,33 @@ orig_vars.pop('__weakref__', None) return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper + + +def raise_error_if_module_name_forbidden(full_module_name): + # it is bad idea to call the pyx-file cython.pyx, so fail early + if full_module_name == 'cython' or full_module_name.startswith('cython.'): + raise ValueError('cython is a special module, cannot be used as a module name') + + +def build_hex_version(version_string): + """ + Parse and translate '4.3a1' into the readable hex representation '0x040300A1' (like PY_VERSION_HEX). + """ + # First, parse '4.12a1' into [4, 12, 0, 0xA01]. + digits = [] + release_status = 0xF0 + for digit in re.split('([.abrc]+)', version_string): + if digit in ('a', 'b', 'rc'): + release_status = {'a': 0xA0, 'b': 0xB0, 'rc': 0xC0}[digit] + digits = (digits + [0, 0])[:3] # 1.2a1 -> 1.2.0a1 + elif digit != '.': + digits.append(int(digit)) + digits = (digits + [0] * 3)[:4] + digits[3] += release_status + + # Then, build a single hex value, two hex digits per version part. + hexversion = 0 + for digit in digits: + hexversion = (hexversion << 8) + digit + + return '0x%08X' % hexversion diff -Nru cython-0.20.1+1~201611251650-6686/debian/bzr-builder.manifest cython-0.20.1+1~202203241016-9537/debian/bzr-builder.manifest --- cython-0.20.1+1~201611251650-6686/debian/bzr-builder.manifest 2016-11-25 16:50:58.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/debian/bzr-builder.manifest 2022-03-24 10:16:48.000000000 +0000 @@ -1,3 +1,3 @@ -# bzr-builder format 0.3 deb-version 0.20.1+1~201611251650-6686-pkg8 -lp:~cython-dev/cython/master revid:git-v1:15a40e21bdc53f9421c0d90eb0b54dcf6c796e39 +# bzr-builder format 0.3 deb-version 0.20.1+1~202203241016-9537-pkg8 +lp:~cython-dev/cython/master revid:git-v1:d85cbf676dd7bec80fbfc23d038575a21859ac87 nest-part packaging lp:~cython-dev/cython/packaging py2_and_py3 debian revid:thopiekar@home-20131130113131-9q4vuseaqyfh9sh0 diff -Nru cython-0.20.1+1~201611251650-6686/debian/changelog cython-0.20.1+1~202203241016-9537/debian/changelog --- cython-0.20.1+1~201611251650-6686/debian/changelog 2016-11-25 16:50:58.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/debian/changelog 2022-03-24 10:16:48.000000000 +0000 @@ -1,8 +1,8 @@ -cython (0.20.1+1~201611251650-6686-pkg8~ubuntu14.04.1) trusty; urgency=low +cython (0.20.1+1~202203241016-9537-pkg8~ubuntu14.04.1) trusty; urgency=low * Auto build. - -- Thomas Karl Pietrowski Fri, 25 Nov 2016 16:50:58 +0000 + -- Thomas Karl Pietrowski Thu, 24 Mar 2022 10:16:48 +0000 cython (0.17.x) experimental; urgency=low diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/bpnn3.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/bpnn3.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/bpnn3.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/bpnn3.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,7 @@ #!/usr/bin/python # Back-Propagation Neural Networks -# -# Written in Python. See http://www.python.org/ +# +# Written in Python. See https://www.python.org/ # # Neil Schemenauer @@ -26,10 +26,10 @@ return m class NN(object): -# print 'class NN' +# print 'class NN' def __init__(self, ni, nh, no): # number of input, hidden, and output nodes - self.ni = ni + 1 # +1 for bias node + self.ni = ni + 1 # +1 for bias node self.nh = nh self.no = no @@ -37,11 +37,11 @@ self.ai = [1.0]*self.ni self.ah = [1.0]*self.nh self.ao = [1.0]*self.no - + # create weights self.wi = makeMatrix(self.ni, self.nh) self.wo = makeMatrix(self.nh, self.no) - # set them to random vaules + # set them to random values for i in range(self.ni): for j in range(self.nh): self.wi[i][j] = rand(-2.0, 2.0) @@ -49,7 +49,7 @@ for k in range(self.no): self.wo[j][k] = rand(-2.0, 2.0) - # last change in weights for momentum + # last change in weights for momentum self.ci = makeMatrix(self.ni, self.nh) self.co = makeMatrix(self.nh, self.no) @@ -67,7 +67,7 @@ for j in range(self.nh): sum = 0.0 for i in range(self.ni): - sum = sum + self.ai[i] * self.wi[i][j] + sum = sum + self.ai[i] * self.wi[i][j] self.ah[j] = 1.0/(1.0+math.exp(-sum)) # output activations diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/chaos.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/chaos.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/chaos.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/chaos.py 2022-03-24 10:16:46.000000000 +0000 @@ -130,7 +130,7 @@ I = ii break else: - I = dom[1] - 1 + I = dom[1] - 1 return I def __len__(self): diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/hexiom2.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/hexiom2.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/hexiom2.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/hexiom2.py 2022-03-24 10:16:46.000000000 +0000 @@ -17,7 +17,7 @@ def __init__(self, x, y): self.x = x self.y = y - + DIRS = [ Dir(1, 0), Dir(-1, 0), Dir(0, 1), @@ -35,7 +35,7 @@ FIRST_STRATEGY = 3 MAX_NEIGHBORS_STRATEGY = 4 MIN_NEIGHBORS_STRATEGY = 5 - + def __init__(self, count, empty=False): self.count = count self.cells = None if empty else [[0, 1, 2, 3, 4, 5, 6, EMPTY] for i in range(count)] @@ -60,11 +60,11 @@ return True else: return False - + def remove_all(self, v): for i in range(self.count): self.remove(i, v) - + def remove_unfixed(self, v): changed = False for i in range(self.count): @@ -72,7 +72,7 @@ if self.remove(i, v): changed = True return changed - + def filter_tiles(self, tiles): for v in range(8): if tiles[v] == 0: @@ -206,14 +206,14 @@ def contains_pos(self, pos): return pos in self.nodes_by_pos - + def get_by_pos(self, pos): return self.nodes_by_pos[pos] def get_by_id(self, id): return self.nodes_by_id[id] - + ################################## class Pos(object): def __init__(self, hex, tiles, done = None): @@ -223,7 +223,7 @@ def clone(self): return Pos(self.hex, self.tiles, self.done.clone()) - + ################################## @cython.locals(pos=Pos, i=cython.long, v=cython.int, @@ -260,7 +260,7 @@ for cell in done.cells: if len(cell) == 1: left[cell[0]] -= 1 - + for v in range(8): # If there is none, remove the possibility from all tiles if (pos.tiles[v] > 0) and (left[v] == 0): @@ -276,7 +276,7 @@ if (not done.already_done(i)) and (v in cell): done.set_done(i, v) changed = True - + # Force empty or non-empty around filled cells filled_cells = (range(done.count) if last_move is None else [last_move]) @@ -307,7 +307,7 @@ for u in unknown: if done.remove(u, EMPTY): changed = True - + return changed ASCENDING = 1 @@ -402,7 +402,7 @@ if (not all_done) or (not exact): return OPEN - + print_pos(pos, output) return SOLVED @@ -414,7 +414,7 @@ pass else: pos = prev - + moves = find_moves(pos, strategy, order) if len(moves) == 0: return solved(pos, output) @@ -481,12 +481,12 @@ else: inctile = int(tile) tiles[inctile] += 1 - # Look for locked tiles + # Look for locked tiles if tile[0] == "+": print("Adding locked tile: %d at pos %d, %d, id=%d" % (inctile, x, y, hex.get_by_pos((x, y)).id)) done.set_done(hex.get_by_pos((x, y)).id, inctile) - + linei += 1 for y in range(1, size): ry = size - 1 + y @@ -500,7 +500,7 @@ else: inctile = int(tile) tiles[inctile] += 1 - # Look for locked tiles + # Look for locked tiles if tile[0] == "+": print("Adding locked tile: %d at pos %d, %d, id=%d" % (inctile, x, ry, hex.get_by_pos((x, ry)).id)) @@ -530,13 +530,13 @@ output = StringIO() solve_file(f, strategy, order, output) expected = """\ - 3 4 3 2 - 3 4 4 . 3 - 2 . . 3 4 3 -2 . 1 . 3 . 2 - 3 3 . 2 . 2 - 3 . 2 . 2 - 2 2 . 1 + 3 4 3 2 + 3 4 4 . 3 + 2 . . 3 4 3 +2 . 1 . 3 . 2 + 3 3 . 2 . 2 + 3 . 2 . 2 + 2 2 . 1 """ if output.getvalue() != expected: raise AssertionError("got a wrong answer:\n%s" % output.getvalue()) diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/meteor_contest.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/meteor_contest.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/meteor_contest.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/meteor_contest.py 2022-03-24 10:16:46.000000000 +0000 @@ -64,7 +64,7 @@ def get_puzzle(w=w, h=h): - board = [E*x + S*y + (y%2) for y in range(h) for x in range(w)] + board = [E*x + S*y + (y % 2) for y in range(h) for x in range(w)] cti = dict((board[i], i) for i in range(len(board))) idos = [[E, E, E, SE], # incremental direction offsets @@ -143,7 +143,7 @@ tk = time.time() times.append(tk - t0) return times - + if __name__ == "__main__": parser = optparse.OptionParser( usage="%prog [options]", @@ -152,4 +152,3 @@ options, args = parser.parse_args() util.run_benchmark(options, options.num_runs, main) - diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/nqueens.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/nqueens.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/nqueens.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/nqueens.py 2022-03-24 10:16:46.000000000 +0000 @@ -43,7 +43,7 @@ else: return -# From http://code.activestate.com/recipes/576647/ +# From https://code.activestate.com/recipes/576647/ @cython.locals(queen_count=int, i=int, vec=list) def n_queens(queen_count): """N-Queens solver. diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/richards.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/richards.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/richards.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/richards.py 2022-03-24 10:16:46.000000000 +0000 @@ -102,13 +102,13 @@ self.task_waiting = False self.task_holding = False return self - + def waitingWithPacket(self): self.packet_pending = True self.task_waiting = True self.task_holding = False return self - + def isPacketPending(self): return self.packet_pending @@ -233,7 +233,7 @@ if t is None: raise Exception("Bad task id %d" % id) return t - + # DeviceTask @@ -307,7 +307,7 @@ else: i.control = i.control//2 ^ 0xd008 return self.release(I_DEVB) - + # WorkTask @@ -333,7 +333,7 @@ pkt.ident = dest pkt.datum = 0 - for i in BUFSIZE_RANGE: # range(BUFSIZE) + for i in BUFSIZE_RANGE: # range(BUFSIZE) w.count += 1 if w.count > 26: w.count = 1 @@ -382,10 +382,10 @@ wkq = Packet(wkq , I_DEVB, K_DEV) HandlerTask(I_HANDLERB, 3000, wkq, TaskState().waitingWithPacket(), HandlerTaskRec()) - wkq = None; - DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()); - DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()); - + wkq = None + DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()) + DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()) + schedule() if taskWorkArea.holdCount == 9297 and taskWorkArea.qpktCount == 23246: @@ -431,7 +431,7 @@ for it in item.__dict__.values(): if isinstance(it, types.FunctionType): pypyjit.enable(it.func_code) - + if __name__ == '__main__': import sys if len(sys.argv) >= 2: diff -Nru cython-0.20.1+1~201611251650-6686/Demos/benchmarks/spectralnorm.py cython-0.20.1+1~202203241016-9537/Demos/benchmarks/spectralnorm.py --- cython-0.20.1+1~201611251650-6686/Demos/benchmarks/spectralnorm.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/benchmarks/spectralnorm.py 2022-03-24 10:16:46.000000000 +0000 @@ -11,28 +11,28 @@ import util import optparse -def eval_A (i, j): +def eval_A(i, j): return 1.0 / ((i + j) * (i + j + 1) / 2 + i + 1) -def eval_A_times_u (u): +def eval_A_times_u(u): return [ part_A_times_u(i,u) for i in range(len(u)) ] -def eval_At_times_u (u): +def eval_At_times_u(u): return [ part_At_times_u(i,u) for i in range(len(u)) ] -def eval_AtA_times_u (u): - return eval_At_times_u (eval_A_times_u (u)) +def eval_AtA_times_u(u): + return eval_At_times_u(eval_A_times_u(u)) def part_A_times_u(i, u): partial_sum = 0 for j, u_j in enumerate(u): - partial_sum += eval_A (i, j) * u_j + partial_sum += eval_A(i, j) * u_j return partial_sum def part_At_times_u(i, u): partial_sum = 0 for j, u_j in enumerate(u): - partial_sum += eval_A (j, i) * u_j + partial_sum += eval_A(j, i) * u_j return partial_sum DEFAULT_N = 130 @@ -43,19 +43,19 @@ t0 = time() u = [1] * DEFAULT_N - for dummy in range (10): - v = eval_AtA_times_u (u) - u = eval_AtA_times_u (v) + for dummy in range(10): + v = eval_AtA_times_u(u) + u = eval_AtA_times_u(v) vBv = vv = 0 - for ue, ve in zip (u, v): + for ue, ve in zip(u, v): vBv += ue * ve vv += ve * ve tk = time() times.append(tk - t0) return times - + if __name__ == "__main__": parser = optparse.OptionParser( usage="%prog [options]", diff -Nru cython-0.20.1+1~201611251650-6686/Demos/callback/cheese.pyx cython-0.20.1+1~202203241016-9537/Demos/callback/cheese.pyx --- cython-0.20.1+1~201611251650-6686/Demos/callback/cheese.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/callback/cheese.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,7 +8,7 @@ def find(f): find_cheeses(callback, f) - + cdef void callback(char *name, void *f): (f)(name.decode('utf-8')) - + diff -Nru cython-0.20.1+1~201611251650-6686/Demos/callback/README.rst cython-0.20.1+1~202203241016-9537/Demos/callback/README.rst --- cython-0.20.1+1~201611251650-6686/Demos/callback/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/callback/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +This example demonstrates how you can wrap a C API +that has a callback interface, so that you can +pass Python functions to it as callbacks. + +The files ``cheesefinder.h`` and ``cheesefinder.c`` +represent the C library to be wrapped. + +The file ``cheese.pyx`` is the Cython module +which wraps it. + +The file ``run_cheese.py`` demonstrates how to +call the wrapper. diff -Nru cython-0.20.1+1~201611251650-6686/Demos/callback/README.txt cython-0.20.1+1~202203241016-9537/Demos/callback/README.txt --- cython-0.20.1+1~201611251650-6686/Demos/callback/README.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/callback/README.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -This example demonstrates how you can wrap a C API -that has a callback interface, so that you can -pass Python functions to it as callbacks. - -The files cheesefinder.h and cheesefinder.c -represent the C library to be wrapped. - -The file cheese.pyx is the Pyrex module -which wraps it. - -The file run_cheese.py demonstrates how to -call the wrapper. diff -Nru cython-0.20.1+1~201611251650-6686/Demos/callback/run_cheese.py cython-0.20.1+1~202203241016-9537/Demos/callback/run_cheese.py --- cython-0.20.1+1~201611251650-6686/Demos/callback/run_cheese.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/callback/run_cheese.py 2022-03-24 10:16:46.000000000 +0000 @@ -4,5 +4,3 @@ print("Found cheese: " + name) cheese.find(report_cheese) - - diff -Nru cython-0.20.1+1~201611251650-6686/Demos/embed/Makefile cython-0.20.1+1~202203241016-9537/Demos/embed/Makefile --- cython-0.20.1+1~201611251650-6686/Demos/embed/Makefile 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/embed/Makefile 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,7 @@ # Makefile for creating our standalone Cython program PYTHON := python PYVERSION := $(shell $(PYTHON) -c "import sys; print(sys.version[:3])") +PYPREFIX := $(shell $(PYTHON) -c "import sys; print(sys.prefix)") INCDIR := $(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc())") PLATINCDIR := $(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc(plat_specific=True))") @@ -14,6 +15,23 @@ LIBS := $(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LIBS'))") SYSLIBS := $(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SYSLIBS'))") +.PHONY: paths all clean test + +paths: + @echo "PYTHON=$(PYTHON)" + @echo "PYVERSION=$(PYVERSION)" + @echo "PYPREFIX=$(PYPREFIX)" + @echo "INCDIR=$(INCDIR)" + @echo "PLATINCDIR=$(PLATINCDIR)" + @echo "LIBDIR1=$(LIBDIR1)" + @echo "LIBDIR2=$(LIBDIR2)" + @echo "PYLIB=$(PYLIB)" + @echo "CC=$(CC)" + @echo "LINKCC=$(LINKCC)" + @echo "LINKFORSHARED=$(LINKFORSHARED)" + @echo "LIBS=$(LIBS)" + @echo "SYSLIBS=$(SYSLIBS)" + embedded: embedded.o $(LINKCC) -o $@ $^ -L$(LIBDIR1) -L$(LIBDIR2) -l$(PYLIB) $(LIBS) $(SYSLIBS) $(LINKFORSHARED) diff -Nru cython-0.20.1+1~201611251650-6686/Demos/embed/README cython-0.20.1+1~202203241016-9537/Demos/embed/README --- cython-0.20.1+1~201611251650-6686/Demos/embed/README 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/embed/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -This example demonstrates how Cython-generated code -can be called directly from a main program written in C. - -The Windows makefiles were contributed by -Duncan Booth . diff -Nru cython-0.20.1+1~201611251650-6686/Demos/embed/README.rst cython-0.20.1+1~202203241016-9537/Demos/embed/README.rst --- cython-0.20.1+1~201611251650-6686/Demos/embed/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/embed/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,5 @@ +This example demonstrates how Cython-generated code +can be called directly from a main program written in C. + +The Windows makefiles were contributed by +Duncan Booth: Duncan.Booth@SuttonCourtenay.org.uk. diff -Nru cython-0.20.1+1~201611251650-6686/Demos/freeze/README.rst cython-0.20.1+1~202203241016-9537/Demos/freeze/README.rst --- cython-0.20.1+1~201611251650-6686/Demos/freeze/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/freeze/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,111 @@ +NAME +==== + +**cython_freeze** - create a C file for embedding Cython modules + + +SYNOPSIS +======== +:: + + cython_freeze [-o outfile] [-p] module [...] + + +DESCRIPTION +=========== + +**cython_freeze** generates a C source file to embed a Python interpreter +with one or more Cython modules built in. This allows one to create a single +executable from Cython code, without having to have separate shared objects +for each Cython module. A major advantage of this approach is that it allows +debugging with gprof(1), which does not work with shared objects. + +Unless ``-p`` is given, the first module's ``__name__`` is set to +``"__main__"`` and is imported on startup; if ``-p`` is given, a normal Python +interpreter is built, with the given modules built into the binary. + +Note that this method differs from ``cython --embed``. The ``--embed`` options +modifies the resulting C source file to include a ``main()`` function, so it +can only be used on a single Cython module. The advantage ``--embed`` is +simplicity. This module, on the other hand, can be used with multiple +modules, but it requires another C source file to be created. + + +OPTIONS +======= +:: + + -o FILE, --outfile=FILE write output to FILE instead of standard output + -p, --pymain do not automatically run the first module as __main__ + + +EXAMPLE +======= + +In the ``Demos/freeze`` directory, there exist two Cython modules: + +* ``lcmath.pyx``: A module that interfaces with the -lm library. + +* ``combinatorics.pyx``: A module that implements n-choose-r using lcmath. + +Both modules have the Python idiom ``if __name__ == "__main__"``, which only +execute if that module is the "main" module. If run as main, lcmath prints the +factorial of the argument, while combinatorics prints n-choose-r. + +The provided Makefile creates an executable, *nCr*, using combinatorics as the +"main" module. It basically performs the following (ignoring the compiler +flags):: + + $ cython_freeze combinatorics lcmath > nCr.c + $ cython combinatorics.pyx + $ cython lcmath.pyx + $ gcc -c nCr.c + $ gcc -c combinatorics.c + $ gcc -c lcmath.c + $ gcc nCr.o combinatorics.o lcmath.o -o nCr + +Because the combinatorics module was listed first, its ``__name__`` is set +to ``"__main__"``, while lcmath's is set to ``"lcmath"``. The executable now +contains a Python interpreter and both Cython modules. :: + + $ ./nCr + USAGE: ./nCr n r + Prints n-choose-r. + $ ./nCr 15812351235 12 + 5.10028093999e+113 + +You may wish to build a normal Python interpreter, rather than having one +module as "main". This may happen if you want to use your module from an +interactive shell or from another script, yet you still want it statically +linked so you can profile it with gprof. To do this, add the ``--pymain`` +flag to ``cython_freeze``. In the Makefile, the *python* executable is built +like this. :: + + $ cython_freeze --pymain combinatorics lcmath -o python.c + $ gcc -c python.c + $ gcc python.o combinatorics.o lcmath.o -o python + +Now ``python`` is a normal Python interpreter, but the lcmath and combinatorics +modules will be built into the executable. :: + + $ ./python + Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18) + [GCC 4.3.3] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import lcmath + >>> lcmath.factorial(155) + 4.7891429014634364e+273 + + +PREREQUISITES +============= + +Cython 0.11.2 (or newer, assuming the API does not change) + + +SEE ALSO +======== + +* `Python `_ +* `Cython `_ +* `freeze.py `_ diff -Nru cython-0.20.1+1~201611251650-6686/Demos/freeze/README.txt cython-0.20.1+1~202203241016-9537/Demos/freeze/README.txt --- cython-0.20.1+1~201611251650-6686/Demos/freeze/README.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/freeze/README.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,111 +0,0 @@ -NAME -==== - -cython_freeze - create a C file for embedding Cython modules - - -SYNOPSIS -======== - -cython_freeze [-o outfile] [-p] module [...] - - -DESCRIPTION -=========== - -**cython_freeze** generates a C source file to embed a Python interpreter -with one or more Cython modules built in. This allows one to create a single -executable from Cython code, without having to have separate shared objects -for each Cython module. A major advantage of this approach is that it allows -debuging with gprof(1), which does not work with shared objects. - -Unless ``-p`` is given, the first module's ``__name__`` is set to -``"__main__"`` and is imported on startup; if ``-p`` is given, a normal Python -interpreter is built, with the given modules built into the binary. - -Note that this method differs from ``cython --embed``. The ``--embed`` options -modifies the resulting C source file to include a ``main()`` function, so it -can only be used on a single Cython module. The advantage ``--embed`` is -simplicity. This module, on the other hand, can be used with multiple -modules, but it requires another C source file to be created. - - -OPTIONS -======= - --o FILE, --outfile=FILE write output to FILE instead of standard output --p, --pymain do not automatically run the first module as __main__ - - -EXAMPLE -======= - -In the Demos/freeze directory, there exist two Cython modules: - -lcmath.pyx - A module that interfaces with the -lm library. - -combinatorics.pyx - A module that implements n-choose-r using lcmath. - -Both modules have the Python idiom ``if __name__ == "__main__"``, which only -execute if that module is the "main" module. If run as main, lcmath prints the -factorial of the argument, while combinatorics prints n-choose-r. - -The provided Makefile creates an executable, *nCr*, using combinatorics as the -"main" module. It basically performs the following (ignoring the compiler -flags):: - - $ cython_freeze combinatorics lcmath > nCr.c - $ cython combinatorics.pyx - $ cython lcmath.pyx - $ gcc -c nCr.c - $ gcc -c combinatorics.c - $ gcc -c lcmath.c - $ gcc nCr.o combinatorics.o lcmath.o -o nCr - -Because the combinatorics module was listed first, its ``__name__`` is set -to ``"__main__"``, while lcmath's is set to ``"lcmath"``. The executable now -contains a Python interpreter and both Cython modules. :: - - $ ./nCr - USAGE: ./nCr n r - Prints n-choose-r. - $ ./nCr 15812351235 12 - 5.10028093999e+113 - -You may wish to build a normal Python interpreter, rather than having one -module as "main". This may happen if you want to use your module from an -interactive shell or from another script, yet you still want it statically -linked so you can profile it with gprof. To do this, add the ``--pymain`` -flag to ``cython_freeze``. In the Makefile, the *python* executable is built -like this. :: - - $ cython_freeze --pymain combinatorics lcmath -o python.c - $ gcc -c python.c - $ gcc python.o combinatorics.o lcmath.o -o python - -Now ``python`` is a normal Python interpreter, but the lcmath and combinatorics -modules will be built into the executable. :: - - $ ./python - Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18) - [GCC 4.3.3] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import lcmath - >>> lcmath.factorial(155) - 4.7891429014634364e+273 - - -PREREQUISITES -============= - -Cython 0.11.2 (or newer, assuming the API does not change) - - -SEE ALSO -======== - -* `Python `_ -* `Cython `_ -* `freeze.py `_ diff -Nru cython-0.20.1+1~201611251650-6686/Demos/libraries/setup.py cython-0.20.1+1~202203241016-9537/Demos/libraries/setup.py --- cython-0.20.1+1~201611251650-6686/Demos/libraries/setup.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/libraries/setup.py 2022-03-24 10:16:46.000000000 +0000 @@ -18,7 +18,7 @@ print("Error building external library, please create libmymath.a manually.") sys.exit(1) -# Here is how to use the library built above. +# Here is how to use the library built above. ext_modules = cythonize([ Extension("call_mymath", sources=["call_mymath.pyx"], diff -Nru cython-0.20.1+1~201611251650-6686/Demos/overflow_perf_run.py cython-0.20.1+1~202203241016-9537/Demos/overflow_perf_run.py --- cython-0.20.1+1~201611251650-6686/Demos/overflow_perf_run.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/overflow_perf_run.py 2022-03-24 10:16:46.000000000 +0000 @@ -16,7 +16,7 @@ print(func.__name__) for type in ['int', 'unsigned int', 'long long', 'unsigned long long', 'object']: if func == most_orthogonal: - if type == 'object' or np == None: + if type == 'object' or np is None: continue type_map = {'int': 'int32', 'unsigned int': 'uint32', 'long long': 'int64', 'unsigned long long': 'uint64'} shape = N, 3 diff -Nru cython-0.20.1+1~201611251650-6686/Demos/pyprimes.py cython-0.20.1+1~202203241016-9537/Demos/pyprimes.py --- cython-0.20.1+1~201611251650-6686/Demos/pyprimes.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/pyprimes.py 2022-03-24 10:16:46.000000000 +0000 @@ -5,9 +5,9 @@ while k < kmax: i = 0 while i < k and n % p[i] != 0: - i = i + 1 + i += 1 if i == k: p.append(n) - k = k + 1 - n = n + 1 + k += 1 + n += 1 return p diff -Nru cython-0.20.1+1~201611251650-6686/Demos/Readme_demos.rst cython-0.20.1+1~202203241016-9537/Demos/Readme_demos.rst --- cython-0.20.1+1~201611251650-6686/Demos/Readme_demos.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/Readme_demos.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,35 +0,0 @@ -* To run demos do:: - - cd Demos - make test - - which runs run_primes.py, run_numeric_demo.py, run_spam.py, - integrate_timing.py, callback/runcheese.py and embed/embedded - -* For other demos:: - - cd libraries - python setup.py build_ext --inplace - python -c 'import call_mymath;print(call_mymath.call_sinc(1))' - - To run one of the benchmarks for 10 iterations to compare cython and python timings:: - - cd benchmarks - python setup.py build_ext --inplace - python nqueens.py -n 10 - python -c 'import nqueens;print(nqueens.test_n_queens(10))' - - To demo cython/bin/cython_freeze:: - - make - ./nCr 10 5 - ./python - -* Build notes - - * benchmarks/chaos.py requires cython 0.24 or newer - - * embed and freeze work for python2, require cython 0.24 or higher - for python 3.5 - - diff -Nru cython-0.20.1+1~201611251650-6686/Demos/README.rst cython-0.20.1+1~202203241016-9537/Demos/README.rst --- cython-0.20.1+1~201611251650-6686/Demos/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Demos/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,35 @@ +To run demos do:: + + cd Demos + make test + +which runs ``run_primes.py``, ``run_numeric_demo.py``, ``run_spam.py``, +``integrate_timing.py``, ``callback/runcheese.py`` and ``embed/embedded`` + +For other demos:: + + cd libraries + python setup.py build_ext --inplace + python -c 'import call_mymath;print(call_mymath.call_sinc(1))' + +To run one of the benchmarks for 10 iterations to compare cython and python timings:: + + cd benchmarks + python setup.py build_ext --inplace + python nqueens.py -n 10 + python -c 'import nqueens;print(nqueens.test_n_queens(10))' + +To demo ``cython/bin/cython_freeze``:: + + make + ./nCr 10 5 + ./python + +* Build notes + + * benchmarks/chaos.py requires cython 0.24 or newer + + * embed and freeze work for python2, require cython 0.24 or higher + for python 3.5 + + diff -Nru cython-0.20.1+1~201611251650-6686/Doc/About.html cython-0.20.1+1~202203241016-9537/Doc/About.html --- cython-0.20.1+1~201611251650-6686/Doc/About.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/About.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,171 +0,0 @@ - - - - - - About Cython - - - -
-

- -
Cython

- -
A language for writing Python extension modules -
- -

-What is Cython all about?

-Cython is a language specially designed for writing Python extension modules. -It's designed to bridge the gap between the nice, high-level, easy-to-use -world of Python and the messy, low-level world of C. -

You may be wondering why anyone would want a special language for this. -Python is really easy to extend using C or C++, isn't it? Why not just -write your extension modules in one of those languages? -

Well, if you've ever written an extension module for Python, you'll -know that things are not as easy as all that. First of all, there is a -fair bit of boilerplate code to write before you can even get off the ground. -Then you're faced with the problem of converting between Python and C data -types. For the basic types such as numbers and strings this is not too -bad, but anything more elaborate and you're into picking Python objects -apart using the Python/C API calls, which requires you to be meticulous -about maintaining reference counts, checking for errors at every step and -cleaning up properly if anything goes wrong. Any mistakes and you have -a nasty crash that's very difficult to debug. -

Various tools have been developed to ease some of the burdens of producing -extension code, of which perhaps SWIG -is the best known. SWIG takes a definition file consisting of a mixture -of C code and specialised declarations, and produces an extension module. -It writes all the boilerplate for you, and in many cases you can use it -without knowing about the Python/C API. But you need to use API calls if -any substantial restructuring of the data is required between Python and -C. -

What's more, SWIG gives you no help at all if you want to create a new -built-in Python type. It will generate pure-Python classes which -wrap (in a slightly unsafe manner) pointers to C data structures, but creation -of true extension types is outside its scope. -

Another notable attempt at making it easier to extend Python is PyInline -, inspired by a similar facility for Perl. PyInline lets you embed pieces -of C code in the midst of a Python file, and automatically extracts them -and compiles them into an extension. But it only converts the basic types -automatically, and as with SWIG,  it doesn't address the creation -of new Python types. -

Cython aims to go far beyond what any of these previous tools provides. -Cython deals with the basic types just as easily as SWIG, but it also lets -you write code to convert between arbitrary Python data structures and -arbitrary C data structures, in a simple and natural way, without knowing -anything about the Python/C API. That's right -- nothing at all! -Nor do you have to worry about reference counting or error checking -- -it's all taken care of automatically, behind the scenes, just as it is -in interpreted Python code. And what's more, Cython lets you define new -built-in Python types just as easily as you can define new classes -in Python. -

Sound too good to be true? Read on and find out how it's done. -

-The Basics of Cython

-The fundamental nature of Cython can be summed up as follows: Cython is -Python with C data types. -

Cython is Python: Almost any piece of Python code is also valid -Cython code. (There are a few limitations, but this approximation will serve -for now.) The Cython compiler will convert it into C code which makes equivalent -calls to the Python/C API. In this respect, Cython is similar to the former -Python2C project (to which I would supply a reference except that it no -longer seems to exist). -

...with C data types. But Cython is much more than that, because -parameters and variables can be declared to have C data types. Code which -manipulates Python values and C values can be freely intermixed, with conversions -occurring automatically wherever possible. Reference count maintenance -and error checking of Python operations is also automatic, and the full -power of Python's exception handling facilities, including the try-except -and try-finally statements, is available to you -- even in the midst of -manipulating C data. -

Here's a small example showing some of what can be done. It's a routine -for finding prime numbers. You tell it how many primes you want, and it -returns them as a Python list. -

primes.pyx
- -
-
 1  def primes(int kmax):
- 2      cdef int n, k, i
- 3      cdef int p[1000]
- 4      result = []
- 5      if kmax > 1000:
- 6          kmax = 1000
- 7      k = 0
- 8      n = 2
- 9      while k < kmax:
-10          i = 0
-11          while i < k and n % p[i] <> 0:
-12              i = i + 1
-13          if i == k:
-14             p[k] = n
-15             k = k + 1
-16             result.append(n)
-17          n = n + 1
-18      return result
-
-You'll see that it starts out just like a normal Python function definition, -except that the parameter kmax is declared to be of type int -. This means that the object passed will be converted to a C integer (or -a TypeError will be raised if it can't be). -

Lines 2 and 3 use the cdef statement to define some local C variables. -Line 4 creates a Python list which will be used to return the result. You'll -notice that this is done exactly the same way it would be in Python. Because -the variable result hasn't been given a type, it is assumed to hold -a Python object. -

Lines 7-9 set up for a loop which will test candidate numbers for primeness -until the required number of primes has been found. Lines 11-12, which -try dividing a candidate by all the primes found so far, are of particular -interest. Because no Python objects are referred to, the loop is translated -entirely into C code, and thus runs very fast. -

When a prime is found, lines 14-15 add it to the p array for fast access -by the testing loop, and line 16 adds it to the result list. Again, you'll -notice that line 16 looks very much like a Python statement, and in fact -it is, with the twist that the C parameter n is automatically converted -to a Python object before being passed to the append method. Finally, -at line 18, a normal Python return statement returns the result -list. -

Compiling primes.pyx with the Cython compiler produces an extension module -which we can try out in the interactive interpreter as follows: -

-
>>> import primes
->>> primes.primes(10)
-[2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
->>>
-
-See, it works! And if you're curious about how much work Cython has saved -you, take a look at the C code generated for this module -. -

-Language Details

-For more about the Cython language, see the Language -Overview . -

-Future Plans

-Cython is not finished. Substantial tasks remaining include: -
    -
  • -Support for certain Python language features which are planned but not -yet implemented. See the Limitations -section of the Language Overview for a current -list.
    • -
- -
    -
  • -C++ support. This could be a very big can of worms - careful thought required -before going there.
    • -
- -
    -
  • -Reading C/C++ header files directly would be very nice, but there are some -severe problems that I will have to find solutions for first, such as what -to do about preprocessor macros. My current thinking is to use a separate -tool to convert .h files into Cython declarations, possibly with some manual -intervention.
    • -
- - - diff -Nru cython-0.20.1+1~201611251650-6686/Doc/extension_types.html cython-0.20.1+1~202203241016-9537/Doc/extension_types.html --- cython-0.20.1+1~201611251650-6686/Doc/extension_types.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/extension_types.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,444 +0,0 @@ - - - - Extension Types - -

Extension Types -

-

Contents

- -

Introduction

- As well as creating normal user-defined classes with the Python class -statement, Cython also lets you create new built-in Python types, known as -extension types. You define an extension type using the cdef class statement. Here's an example: -
cdef class Shrubbery:

    cdef int width, height

-

    def __init__(self, w, h):
-         self.width = w
-         self.height = h

-

    def describe(self):
-         print "This shrubbery is", -self.width, \
-             -"by", self.height, "cubits."

-
- As you can see, a Cython extension type definition looks a lot like a Python - class definition. Within it, you use the def statement to define -methods that can be called from Python code. You can even define many of -the special methods such as __init__ as you would in Python. -

The main difference is that you can use the cdef statement to define -attributes. The attributes may be Python objects (either generic or of a particular -extension type), or they may be of any C data type. So you can use extension -types to wrap arbitrary C data structures and provide a Python-like interface -to them.

-

Attributes

- Attributes of an extension type are stored directly in the object's C struct. - The set of attributes is fixed at compile time; you can't add attributes -to an extension type instance at run time simply by assigning to them, as -you could with a Python class instance. (You can subclass the extension type -in Python and add attributes to instances of the subclass, however.) -

There are two ways that attributes of an extension type can be accessed: - by Python attribute lookup, or by direct access to the C struct from Cython - code. Python code is only able to access attributes of an extension type -by the first method, but Cython code can use either method.

-

By default, extension type attributes are only accessible by direct access, -not Python access, which means that they are not accessible from Python code. -To make them accessible from Python code, you need to declare them as public or readonly. For example,

-
cdef class Shrubbery:
-     cdef public int width, height
-     cdef readonly float depth
- makes the width and height attributes readable and writable - from Python code, and the depth attribute readable but not writable. - -

Note that you can only expose simple C types, such as ints, floats and - strings, for Python access. You can also expose Python-valued attributes, - although read-write exposure is only possible for generic Python attributes - (of type object). If the attribute is declared to be of an extension - type, it must be exposed readonly.

-

Note also that the public and readonly options apply - only to Python access, not direct access. All the attributes of an -extension type are always readable and writable by direct access.

-

Howerver, for direct access to be possible, the Cython compiler must know -that you have an instance of that type, and not just a generic Python object. -It knows this already in the case of the "self" parameter of the methods of -that type, but in other cases you will have to tell it by means of a declaration. -For example,

-
cdef widen_shrubbery(Shrubbery sh, extra_width):
-     sh.width = sh.width + extra_width
- If you attempt to access an extension type attribute through a generic -object reference, Cython will use a Python attribute lookup. If the attribute -is exposed for Python access (using public or readonly) -then this will work, but it will be much slower than direct access. -

Extension types and None

- When you declare a parameter or C variable as being of an extension type, - Cython will allow it to take on the value None as well as values of its declared -type. This is analogous to the way a C pointer can take on the value NULL, -and you need to exercise the same caution because of it. There is no problem -as long as you are performing Python operations on it, because full dynamic -type checking will be applied. However, when you access C attributes of an -extension type (as in the widen_shrubbery function above), it's up -to you to make sure the reference you're using is not None -- in the interests -of efficiency, Cython does not check this. -

You need to be particularly careful when exposing Python functions which - take extension types as arguments. If we wanted to make widen_shrubbery -a Python function, for example, if we simply wrote

-
def widen_shrubbery(Shrubbery sh, extra_width): # This is
-     sh.width = sh.width + extra_width           -# dangerous!
- then users of our module could crash it by passing None for the sh -parameter. -

One way to fix this would be

-
def widen_shrubbery(Shrubbery sh, extra_width):
-     if sh is None:
-         raise TypeError
-     sh.width = sh.width + extra_width
- but since this is anticipated to be such a frequent requirement, Cython -provides a more convenient way. Parameters of a Python function declared -as an extension type can have a not None clause: -
def widen_shrubbery(Shrubbery sh not None, extra_width): -
-     sh.width = sh.width + extra_width
- Now the function will automatically check that sh is not None -along with checking that it has the right type. -

Note, however that the not None clause can only be used - in Python functions (defined with def) and not C functions (defined - with cdef). If you need to check whether a parameter to a C function - is None, you will need to do it yourself.

-

Some more things to note:

-
    -
  • The self parameter of a method of an extension type is guaranteed - never to be None.
    • -
-
    -
  • When comparing a value with None, keep in mind that, if x is a Python object, x is None and x is not None are very -efficient because they translate directly to C pointer comparisons, whereas - x == None and x != None, or simply using x as a boolean value (as in if x: ...) will invoke Python operations -and therefore be much slower.
    • -
-

Special methods

- Although the principles are similar, there are substantial differences -between many of the __xxx__ special methods of extension types and their -Python counterparts. There is a separate page devoted to this subject, and you should read it carefully before attempting -to use any special methods in your extension types. -

Properties

- There is a special syntax for defining properties in an extension - class: -
cdef class Spam:

    property cheese:

-

        "A doc string can go -here."

-

        def __get__(self): -
-             -# This is called when the property is read.
-             -...

-

        def __set__(self, value): -
-             -# This is called when the property is written.
-             -...

-

        def __del__(self): -
-             -# This is called when the property is deleted.
-  

-
- The __get__, __set__ and __del__ methods are -all optional; if they are omitted, an exception will be raised when the corresponding -operation is attempted. -

Here's a complete example. It defines a property which adds to a list -each time it is written to, returns the list when it is read, and empties -the list when it is deleted.
-  

-
- - - - - - - - - - - - - - - -
cheesy.pyxTest input
cdef class CheeseShop: -

  cdef object cheeses

-

  def __new__(self):
-     self.cheeses = []

-

  property cheese:

-

    def __get__(self):
-       return "We don't have: %s" % self.cheeses -

-

    def __set__(self, value):
-       self.cheeses.append(value) -

-

    def __del__(self):
-       del self.cheeses[:]

- 		from cheesy import CheeseShop -

shop = CheeseShop()
- print shop.cheese

-

shop.cheese = "camembert"
- print shop.cheese

-

shop.cheese = "cheddar"
- print shop.cheese

-

del shop.cheese
- print shop.cheese

-
Test output
We don't have: []
- We don't have: ['camembert']
- We don't have: ['camembert', 'cheddar']
- We don't have: []
-
-

Subclassing

- An extension type may inherit from a built-in type or another extension -type: -
cdef class Parrot:
-     ...

cdef class Norwegian(Parrot):
-     ...

-
-


- A complete definition of the base type must be available to Cython, so if -the base type is a built-in type, it must have been previously declared as -an extern extension type. If the base type is defined in another Cython -module, it must either be declared as an extern extension type or imported -using the cimport statement.

-

An extension type can only have one base class (no multiple inheritance). -

-

Cython extension types can also be subclassed in Python. A Python class - can inherit from multiple extension types provided that the usual Python -rules for multiple inheritance are followed (i.e. the C layouts of all the -base classes must be compatible).
-

-

C methods

- Extension types can have C methods as well as Python methods. Like C functions, -C methods are declared using cdef instead of def. C methods -are "virtual", and may be overridden in derived extension types.
-
- - - - - - - - - - -
pets.pyx
- 		Output
-
cdef class Parrot:
-
-   cdef void describe(self):
-     print "This parrot is resting."
-
- cdef class Norwegian(Parrot):
-
-   cdef void describe(self):
-    Parrot.describe(self)
-     print "Lovely plumage!"
-
-
- cdef Parrot p1, p2
- p1 = Parrot()
- p2 = Norwegian()
-print "p1:"
- p1.describe()
-print "p2:"
- p2.describe()
- 		p1:
-This parrot is resting.
-p2:
- This parrot is resting.
- Lovely plumage!
-
-
- The above example also illustrates that a C method can call an inherited -C method using the usual Python technique, i.e.
-
Parrot.describe(self)
-
-

Forward-declaring extension types

- Extension types can be forward-declared, like struct and union types. This - will be necessary if you have two extension types that need to refer to -each other, e.g. -
cdef class Shrubbery # forward declaration

cdef class Shrubber:
-     cdef Shrubbery work_in_progress

-

cdef class Shrubbery:
-     cdef Shrubber creator

-
- If you are forward-declaring an exension type that has a base class, you -must specify the base class in both the forward declaration and its subsequent -definition, for example,
-
cdef class A(B)
-
-...
-
-cdef class A(B):
-    # attributes and methods
-
-

Making extension types weak-referenceable

By -default, extension types do not support having weak references made to -them. You can enable weak referencing by declaring a C attribute of -type object called __weakref__. For example,
-
-
cdef class ExplodingAnimal:
-    """This animal will self-destruct when it is
-       no longer strongly referenced."""
-   
-    cdef object __weakref__
-
-
-

Public and external extension types

- - Extension types can be declared extern or public. An extern extension type declaration makes -an extension type defined in external C code available to a Cython module. -A public extension type declaration makes an extension type defined in a Cython module available to external C -code. -

External extension types

- An extern extension type allows you to gain access to the internals - of Python objects defined in the Python core or in a non-Cython extension -module. -
NOTE: In Cython versions before 0.8, extern extension - types were also used to reference extension types defined in another Cython - module. While you can still do that, Cython 0.8 and later provides a better - mechanism for this. See Sharing C Declarations Between - Cython Modules.
- Here is an example which will let you get at the C-level members of the -built-in complex object. -
cdef extern from "complexobject.h":

    struct Py_complex:
-         double real
-         double imag

-

    ctypedef class __builtin__.complex [object PyComplexObject]: -
-         cdef Py_complex cval -

-

# A function which uses the above type
- def spam(complex c):
-     print "Real:", c.cval.real
-     print "Imag:", c.cval.imag

-
- Some important things to note are: -
    -
  1. In this example, ctypedef class has been used. This is because, - in the Python header files, the PyComplexObject struct is declared - with
    -
    -
    ctypedef struct {
    -     ...
    - } PyComplexObject;
    -
    -
    -
  2. As well as the name of the extension type, the module in which -its type object can be found is also specified. See the implicit importing section below. 
    -
    -
    3. -
  3. When declaring an external extension type, you don't declare -any methods. Declaration of methods is not required in order to call them, -because the calls are Python method calls. Also, as with structs and unions, -if your extension class declaration is inside a cdef extern from block, - you only need to declare those C members which you wish to access.
    4. -
-

Implicit importing

-
Backwards Incompatibility Note: -You will have to update any pre-0.8 Cython modules you have which use extern -extension types. I apologise for this, but for complicated reasons it proved - to be too difficult to continue supporting the old way of doing these while - introducing the new features that I wanted.
- Cython 0.8 and later requires you to include a module name in an extern -extension class declaration, for example, -
cdef extern class MyModule.Spam:
-     ...
- The type object will be implicitly imported from the specified module and - bound to the corresponding name in this module. In other words, in this -example an implicit -
    - 
    from MyModule import Spam
    -
- statement will be executed at module load time. -

The module name can be a dotted name to refer to a module inside a package - hierarchy, for example,

-
cdef extern class My.Nested.Package.Spam:
-     ...
- You can also specify an alternative name under which to import the type -using an as clause, for example, -
    - cdef extern class My.Nested.Package.Spam as Yummy:
    -    ...
- which corresponds to the implicit import statement -
    - 
    from My.Nested.Package import Spam as Yummy
    -
-

Type names vs. constructor names

- Inside a Cython module, the name of an extension type serves two distinct - purposes. When used in an expression, it refers to a module-level global -variable holding the type's constructor (i.e. its type-object). However, -it can also be used as a C type name to declare variables, arguments and -return values of that type. -

When you declare

-
cdef extern class MyModule.Spam:
-     ...
- the name Spam serves both these roles. There may be other names - by which you can refer to the constructor, but only Spam can be -used as a type name. For example, if you were to explicity import MyModule, - you could use MyModule.Spam() to create a Spam instance, but you - wouldn't be able to use MyModule.Spam as a type name. -

When an as clause is used, the name specified in the as -clause also takes over both roles. So if you declare

-
cdef extern class MyModule.Spam as Yummy:
-     ...
- then Yummy becomes both the type name and a name for the constructor. - Again, there are other ways that you could get hold of the constructor, -but only Yummy is usable as a type name. -

Public extension types

- An extension type can be declared public, in which case a .h -file is generated containing declarations for its object struct and type -object. By including the .h file in external C code that you write, -that code can access the attributes of the extension type. -

Name specification clause

- The part of the class declaration in square brackets is a special feature - only available for extern or public extension types. The full -form of this clause is -
[object object_struct_name, type type_object_name ]
- where object_struct_name is the name to assume for the type's C -struct, and type_object_name is the name to assume for the type's -statically declared type object. (The object and type clauses can be written -in either order.) -

If the extension type declaration is inside a cdef extern from -block, the object clause is required, because Cython must be able to -generate code that is compatible with the declarations in the header file. -Otherwise, for extern extension types, the object clause is -optional.

-

For public extension types, the object and type clauses -are both required, because Cython must be able to generate code that is compatible -with external C code.

-

-

- Back to the Language Overview
-  
-
- \ No newline at end of file diff -Nru cython-0.20.1+1~201611251650-6686/Doc/FAQ.html cython-0.20.1+1~202203241016-9537/Doc/FAQ.html --- cython-0.20.1+1~201611251650-6686/Doc/FAQ.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/FAQ.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,77 +0,0 @@ - - - - FAQ.html - - -

Cython FAQ -

-
-

Contents

- -

How do I call Python/C API routines?

- Declare them as C functions inside a cdef extern from block. -Use the type name object for any parameters and return types which -are Python object references. Don't use the word const anywhere. -Here is an example which defines and uses the PyString_FromStringAndSize routine: -
cdef extern from "Python.h":
-     object PyString_FromStringAndSize(char *, int)

cdef char buf[42]
- my_string = PyString_FromStringAndSize(buf, 42)

-
-

How do I convert a C string containing null -bytes to a Python string?

- Put in a declaration for the PyString_FromStringAndSize API routine - and use that. See How do I call Python/C API - routines?

How do I access the data inside a Numeric - array object?

- Use a cdef extern from block to include the Numeric header file - and declare the array object as an external extension type. The following - code illustrates how to do this: -
cdef extern from "Numeric/arrayobject.h":

    struct PyArray_Descr:
-         int type_num, elsize
-         char type

-

    ctypedef class Numeric.ArrayType [object PyArrayObject]:
-         cdef char *data
-         cdef int nd
-         cdef int *dimensions, -*strides
-         cdef object base -
-         cdef PyArray_Descr *descr
-         cdef int flags
-

-
-

For more information about external extension types, see the "External Extension Types" -section of the "Extension Types" documentation -page.
-

-

Cython says my extension type object has no attribute -'rhubarb', but I know it does. What gives?

-You're probably trying to access it through a reference which Cython thinks -is a generic Python object. You need to tell Cython that it's a reference -to your extension type by means of a declaration,
-for example,
-
cdef class Vegetables:
-     cdef int rhubarb
-
- ...
- cdef Vegetables veg
- veg.rhubarb = 42
-
-Also see the "Attributes" -section of the "Extension -Types" documentation page.
-

Python says my extension type has no method called 'quack', but I know it does. What gives?

-You may have declared the method using cdef instead of def. Only functions and methods declared with def are callable from Python code.
---- - \ No newline at end of file diff -Nru cython-0.20.1+1~201611251650-6686/Doc/index.html cython-0.20.1+1~202203241016-9537/Doc/index.html --- cython-0.20.1+1~201611251650-6686/Doc/index.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/index.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,74 +0,0 @@ - - - - - - Cython - Front Page - - -  - - - - - - -
CythonA -smooth blend of the finest Python  -
with the unsurpassed power  -
of raw C.
- -
Welcome to Cython, a language for writing Python -extension modules. Cython makes creating an extension module is almost as -easy as creating a Python module! To find out more, consult one of the -edifying documents below.
- -

-Documentation

- -
-

-About Cython

- -
Read this to find out what Cython is all about -and what it can do for you.
- -

-Language -Overview

- -
A description of all the features of the Cython -language. This is the closest thing to a reference manual in existence -yet.
- -

-FAQ

- -
Want to know how to do something in Cython? Check -here first.
-
- -

-Other Resources

- -
-

-Michael's -Quick Guide to Cython

- -
This tutorial-style presentation will take you -through the steps of creating some Cython modules to wrap existing C libraries. -Contributed by Michael JasonSmith.
- -

-Mail -to the Author

- -
If you have a question that's not answered by -anything here, you're not sure about something, or you have a bug to report -or a suggestion to make, or anything at all to say about Cython, feel free -to email me: greg@cosc.canterbury.ac.nz
-
- - - diff -Nru cython-0.20.1+1~201611251650-6686/Doc/overview.html cython-0.20.1+1~202203241016-9537/Doc/overview.html --- cython-0.20.1+1~201611251650-6686/Doc/overview.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/overview.html 1970-01-01 00:00:00.000000000 +0000 @@ -1,960 +0,0 @@ - - - - - - - - Cython Language Overview - - - -

Overview of the Cython Language 

- - This document informally describes the extensions to the Python language - made by Cython. Some day there will be a reference manual covering everything - in more detail.
- -   -

Contents

- - - - - -

Basics -

- - This section describes the basic features of the Cython language. The facilities - covered in this section allow you to create Python-callable functions that - manipulate C data structures and convert between Python and C data types. - Later sections will cover facilities for wrapping external C code, creating new Python types and cooperation between Cython modules. -

Python functions vs. C functions

- - There are two kinds of function definition in Cython: -

Python functions are defined using the def statement, as - in Python. They take Python objects as parameters and return Python objects. -

- - -

C functions are defined using the new cdef statement. They - take either Python objects or C values as parameters, and can return either - Python objects or C values.

- - -

Within a Cython module, Python functions and C functions can call each other -freely, but only Python functions can be called from outside the module by -interpreted Python code. So, any functions that you want to "export" from - your Cython module must be declared as Python functions using def.

- - -

Parameters of either type of function can be declared to have C data types, - using normal C declaration syntax. For example,

- - -
def spam(int i, char *s):
    ...
- 
cdef int eggs(unsigned long l, float f):
    ...
-
- - When a parameter of a Python function is declared to have a C data type, - it is passed in as a Python object and automatically converted to a C value, - if possible. Automatic conversion is currently only possible for numeric -types and string types; attempting to use any other type for the parameter -of a Python function will result in a compile-time error. -

C functions, on the other hand, can have parameters of any type, since - they're passed in directly using a normal C function call.

- - -

Python objects as parameters and return values

- - If no type is specified for a parameter or return value, it is assumed - to be a Python object. (Note that this is different from the C convention, - where it would default to int.) For example, the following defines - a C function that takes two Python objects as parameters and returns a Python - object: -
cdef spamobjs(x, y):
    ...
-
- - Reference counting for these objects is performed automatically according - to the standard Python/C API rules (i.e. borrowed references are taken as - parameters and a new reference is returned). -

The name object can also be used to explicitly declare something - as a Python object. This can be useful if the name being declared would otherwise -be taken as the name of a type, for example,

- - -
cdef ftang(object int):
    ...
-
- - declares a parameter called int which is a Python object. You -can also use object as the explicit return type of a function, e.g. - -
cdef object ftang(object int):
    ...
-
- - In the interests of clarity, it is probably a good idea to always be explicit - about object parameters in C functions. -

C variable and type definitions

- - The cdef statement is also used to declare C variables, either -local or module-level: -
cdef int i, j, k
cdef float f, g[42], *h
-
- - and C struct, union or enum types: -
cdef struct Grail:
    int age
    float volume
- 
cdef union Food:
    char *spam
    float *eggs
- 
cdef enum CheeseType:
    cheddar, edam, 
    camembert
- 
cdef enum CheeseState:
    hard = 1
    soft = 2
    runny = 3
-
- - There is currently no special syntax for defining a constant, but you -can use an anonymous enum declaration for this purpose, for example, -
cdef enum:
-     tons_of_spam = 3
- - Note that the words struct, union and enum are used only when defining a type, not when referring to it. For example, to declare a variable pointing - to a Grail you would write -
cdef Grail *gp
-
- - and not -
cdef struct Grail *gp # WRONG
-
- - There is also a ctypedef statement for giving names to types, e.g. - -
ctypedef unsigned long ULong
- 
ctypedef int *IntPtr
- -

Automatic type conversions

- -In most situations, automatic conversions will be performed for the -basic numeric and string types when a Python object is used in a -context requiring a C value, or vice versa. The following table -summarises the conversion possibilities.
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
C types
- 		From Python types
- 		To Python types
-
[unsigned] char
-[unsigned] short
- int, long		int, long
- 		int
-
unsigned int
-unsigned long
- [unsigned] long long
- - 		int, long
-
- - 		long
-
- -
float, double, long double
- 		int, long, float
- 		float
-
char *
- 		str
- 		str
-
- -
- -

Caveats when using a Python string in a C context

- -You need to be careful when using a Python string in a context expecting a char *. -In this situation, a pointer to the contents of the Python string is -used, which is only valid as long as the Python string exists. So you -need to make sure that a reference to the original Python string is -held for as long as the C string is needed. If you can't guarantee that -the Python string will live long enough, you will need to copy the C -string.
- -
- -Cython detects and prevents some mistakes of this kind. For instance, if you attempt something like
- -
cdef char *s
s = pystring1 + pystring2
- -then Cython will produce the error message "Obtaining char * from temporary Python value". -The reason is that concatenating the two Python strings produces a new -Python string object that is referenced only by a temporary internal -variable that Cython generates. As soon as the statement has finished, -the temporary variable will be decrefed and the Python string -deallocated, leaving s dangling. Since this code could not possibly work, Cython refuses to compile it.
- -
- -The solution is to assign the result of the concatenation to a Python variable, and then obtain the char * from that, i.e.
- -
cdef char *s
p = pystring1 + pystring2
s = p
- -It is then your responsibility to hold the reference p for as long as necessary.
- -
- -Keep in mind that the rules used to detect such errors are only -heuristics. Sometimes Cython will complain unnecessarily, and sometimes -it will fail to detect a problem that exists. Ultimately, you need to -understand the issue and be careful what you do.
- -
-
- - - -

Scope rules

- - Cython determines whether a variable belongs to a local scope, the module - scope, or the built-in scope completely statically. As with Python, - assigning to a variable which is not otherwise declared implicitly declares - it to be a Python variable residing in the scope where it is assigned. Unlike - Python, however, a name which is referred to but not declared or assigned - is assumed to reside in the builtin scope, not the module scope. -Names added to the module dictionary at run time will not shadow such names. - -

You can use a global statement at the module level to explicitly - declare a name to be a module-level name when there would otherwise not be -any indication of this, for example,

- - -
global __name__
- print __name__
- - Without the global statement, the above would print the name of -the builtins module.
- -
- - Note: A consequence of these rules is that the module-level scope behaves - the same way as a Python local scope if you refer to a variable before assigning - to it. In particular, tricks such as the following will not work -in Cython:
- - -
try:
  x = True
except NameError:
  True = 1
-
- - because, due to the assignment, the True will always be looked up in the - module-level scope. You would have to do something like this instead:
- - -
import __builtin__
try:
  True = __builtin__.True
except AttributeError:
  True = 1
-
- - -
-

Statements and expressions

- - Control structures and expressions follow Python syntax for the most part. - When applied to Python objects, they have the same semantics as in Python - (unless otherwise noted). Most of the Python operators can also be applied - to C values, with the obvious semantics. -

If Python objects and C values are mixed in an expression, conversions - are performed automatically between Python objects and C numeric or string - types.

- - -

Reference counts are maintained automatically for all Python objects, and -all Python operations are automatically checked for errors, with appropriate - action taken.

- - -

Differences between C and Cython -expressions

-There -are some differences in syntax and semantics between C expressions and -Cython expressions, particularly in the area of C constructs which have -no direct equivalent in Python.
- -
    -
  • An integer literal without an L suffix is treated as a C constant, and will be truncated to whatever size your C compiler thinks appropriate. With an L suffix, it will be converted to Python long integer (even if it would be small enough to fit into a C int).
    -
    -
    • -
  • There is no -> operator in Cython. Instead of p->x, - use p.x
    • - -  
  • There is no * operator in Cython. Instead of - *p, use p[0]
    • - -  
  • There is an & operator, with the same semantics - as in C.
    • - -  
  • The null C pointer is called NULL, not 0 (and - NULL is a reserved word).
    • - -  
  • Character literals are written with a c prefix, for -example:
    • -
      - 
      c'X'
      -
    -
  • Type casts are written <type>value , for example:
    • -
      - 
      cdef char *p, float *q
      p = <char*>q
      -
    - Warning: Don't attempt to use a typecast to convert between -Python and C data types -- it won't do the right thing. Leave Cython to perform -the conversion automatically. -
- - -

Integer for-loops

- - You should be aware that a for-loop such as -
for i in range(n):
-     ...
- - won't be very fast, even if i and n are declared as -C integers, because range is a Python function. For iterating over -ranges of integers, Cython has another form of for-loop: -
for i from 0 <= i < n:
-     ...
- - If the loop variable and the lower and upper bounds are all C integers, -this form of loop will be much faster, because Cython will translate it into -pure C code. -

Some things to note about the for-from loop:

- - -
    - -
  • The target expression must be a variable name.
    • -
  • The name between the lower and upper bounds must be the same as -the target name.
    • -
  • The direction of iteration is determined by the relations. If they - are both from the set {<, <=} then it is upwards; - if they are both from the set {>, >=} then it is -downwards. (Any other combination is disallowed.)
    • - -
- - Like other Python looping statements, break and continue may be used in the body, and the loop may have an else clause. - -

- - -

Error return values

- - If you don't do anything special, a function declared with cdef that does not return a Python object has no way of reporting Python exceptions - to its caller. If an exception is detected in such a function, a warning -message is printed and the exception is ignored. -

If you want a C function that does not return a Python object to be able - to propagate exceptions to its caller, you need to declare an exception - value for it. Here is an example:

- - -
cdef int spam() except -1:
-     ...
- - With this declaration, whenever an exception occurs inside spam, - it will immediately return with the value -1. Furthermore, whenever - a call to spam returns -1, an exception will be assumed - to have occurred and will be propagated. -

When you declare an exception value for a function, you should never explicitly - return that value. If all possible return values are legal and you can't -reserve one entirely for signalling errors, you can use an alternative form -of exception value declaration:

- - -
cdef int spam() except? -1:
-     ...
- - The "?" indicates that the value -1 only indicates a possible error. In this case, Cython generates a call to PyErr_Occurredif the -exception value is returned, to make sure it really is an error. -

There is also a third form of exception value declaration:

- - -
cdef int spam() except *:
-     ...
- - This form causes Cython to generate a call to PyErr_Occurred after - every call to spam, regardless of what value it returns. If you have - a function returning void that needs to propagate errors, you will - have to use this form, since there isn't any return value to test. -

Some things to note:

- - -
    - -
  • Currently, exception values can only declared for functions returning - an integer, float or pointer type, and the value must be a literal, - not an expression (although it can be negative). The only possible pointer - exception value is NULL. Void functions can only use the except - * form.
    • -
    -  
  • The exception value specification is part of the signature -of the function. If you're passing a pointer to a function as a parameter -or assigning it to a variable, the declared type of the parameter or variable - must have the same exception value specification (or lack thereof). Here -is an example of a pointer-to-function declaration with an exception value:
    • -
      - 
      int (*grail)(int, char *) except -1
      -
    -
  • You don't need to (and shouldn't) declare exception values for functions - which return Python objects. Remember that a function with no declared return - type implicitly returns a Python object.
    • - -
- - -

Checking return values of non-Cython - functions

- - It's important to understand that the except clause does not cause an error to be raised when the specified value is returned. For -example, you can't write something like -
cdef extern FILE *fopen(char *filename, char *mode) except NULL # WRONG!
-
- - and expect an exception to be automatically raised if a call to fopen -returns NULL. The except clause doesn't work that way; its only purpose -is for propagating exceptions that have already been raised, either -by a Cython function or a C function that calls Python/C API routines. To -get an exception from a non-Python-aware function such as fopen, you will -have to check the return value and raise it yourself, for example, -
cdef FILE *p
p = fopen("spam.txt", "r")
if p == NULL:
    raise SpamError("Couldn't open the spam file")
-
- - -

- - -

The include statement

- - For convenience, a large Cython module can be split up into a number of -files which are put together using the include statement, for example - -
include "spamstuff.pxi"
-
- - The contents of the named file are textually included at that point. The - included file can contain any complete top-level Cython statements, including - other include statements. The include statement itself can -only appear at the top level of a file. -

The include statement can also be used in conjunction with public declarations to make C functions and - variables defined in one Cython module accessible to another. However, note - that some of these uses have been superseded by the facilities described -in Sharing Declarations Between Cython Modules, -and it is expected that use of the include statement for this purpose -will be phased out altogether in future versions.

- - -

Interfacing with External - C Code -

- - One of the main uses of Cython is wrapping existing libraries of C code. -This is achieved by using external declarations to declare the C functions and variables from the library that you want to - use. -

You can also use public declarations to make - C functions and variables defined in a Cython module available to external - C code. The need for this is expected to be less frequent, but you might -want to do it, for example, if you are embedding Python in another application - as a scripting language. Just as a Cython module can be used as a bridge to -allow Python code to call C code, it can also be used to allow C code to -call Python code.

- - -

External declarations

- - By default, C functions and variables declared at the module level are -local to the module (i.e. they have the C static storage class). They -can also be declared extern to specify that they are defined elsewhere, - for example: -
cdef extern int spam_counter
- 
cdef extern void order_spam(int tons)
-
- - -
- - -

Referencing C header files

- - When you use an extern definition on its own as in the examples above, -Cython includes a declaration for it in the generated C file. This can cause -problems if the declaration doesn't exactly match the declaration that will -be seen by other C code. If you're wrapping an existing C library, for example, -it's important that the generated C code is compiled with exactly the same -declarations as the rest of the library. -

To achieve this, you can tell Cython that the declarations are to be found - in a C header file, like this:

- - -
cdef extern from "spam.h":
- 
    int spam_counter
- 
    void order_spam(int tons)
-
- - The cdef extern from clause does three things: -
    - -
  1. It directs Cython to place a #include statement for the named - header file in the generated C code.
    -
    2. -  
  2. It prevents Cython from generating any C code for the declarations - found in the associated block.
    -
    3. -  
  3. It treats all declarations within the block as though they -started with cdef extern.
    4. - -
- - It's important to understand that Cython does not itself read the -C header file, so you still need to provide Cython versions of any declarations - from it that you use. However, the Cython declarations don't always have to -exactly match the C ones, and in some cases they shouldn't or can't. In particular: - -
    - -
  1. Don't use const. Cython doesn't know anything about const, -so just leave it out. Most of the time this shouldn't cause any problem, -although on rare occasions you might have to use a cast. 1
    -
    2. -  
  2. Leave out any platform-specific extensions to C declarations - such as __declspec().
    -
    3. -  
  3. If the header file declares a big struct and you only want -to use a few members, you only need to declare the members you're interested -in. Leaving the rest out doesn't do any harm, because the C compiler will -use the full definition from the header file.
    -
    - In some cases, you might not need any of the struct's members, in -which case you can just put pass in the body of the struct declaration, -e.g.
    -
    -     cdef extern from "foo.h":
    -         struct spam:
    -             pass
    -
    -Note that you can only do this inside a cdef extern from block; struct -declarations anywhere else must be non-empty.
    -
    -
    4. -
  4. If the header file uses typedef names such as size_t to refer -to platform-dependent flavours of numeric types, you will need a corresponding - ctypedef statement, but you don't need to match the type exactly, - just use something of the right general kind (int, float, etc). For example,
    5. -
      - 
      ctypedef int size_t
      -
    - will work okay whatever the actual size of a size_t is (provided the header - file defines it correctly).
    -  
  5. If the header file uses macros to define constants, translate - them into a dummy enum declaration.
    -
    6. -  
  6. If the header file defines a function using a macro, declare - it as though it were an ordinary function, with appropriate argument and -result types.
    7. - -
- - A few more tricks and tips: -
    - -
  • If you want to include a C header because it's needed by another -header, but don't want to use any declarations from it, put pass in the extern-from block:
    • - -
- - -
    - -
      - cdef extern from "spam.h":
      -     pass
    - -
- - -
    - -
  • If you want to include some external declarations, but don't want -to specify a header file (because it's included by some other header that -you've already included) you can put * in place of the header file -name:
    • - -
- - -
cdef extern from *:
-     ...
-
- - -

Styles of struct, union and enum declaration

- - There are two main ways that structs, unions and enums can be declared -in C header files: using a tag name, or using a typedef. There are also some - variations based on various combinations of these. -

It's important to make the Cython declarations match the style used in the -header file, so that Cython can emit the right sort of references to the type -in the code it generates. To make this possible, Cython provides two different -syntaxes for declaring a struct, union or enum type. The style introduced -above corresponds to the use of a tag name. To get the other style, you prefix -the declaration with ctypedef, as illustrated below.

- - -

The following table shows the various possible styles that can be found - in a header file, and the corresponding Cython declaration that you should - put in the cdef exern from block. Struct declarations are used as -an example; the same applies equally to union and enum declarations.

- - -

Note that in all the cases below, you refer to the type in Cython code simply -as Foo, not struct Foo. -
-   - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 C codePossibilities for corresponding -Cython codeComments
1struct Foo {
-   ...
- };		cdef struct Foo:
-   ...		Cython will refer to the type as struct Foo in the generated - C code.
2typedef struct {
-   ...
- } Foo;		ctypedef struct Foo:
-   ...		Cython will refer to the type simply as Foo -in the generated C code.
3typedef struct -foo {
-   ...
- } Foo;		cdef struct foo:
-   ...
- ctypedef foo Foo #optional		If the C header uses both a tag and a typedef - with different names, you can use either form of declaration in Cython - (although if you need to forward reference the type, you'll have to use -the first form).
ctypedef struct Foo:
-   ...
4typedef struct Foo {
-   ...
- } Foo;		cdef struct Foo:
-   ...		If the header uses the same name for the tag and the typedef, - you won't be able to include a ctypedef for it -- but then, it's not -necessary.
-

- - -

Accessing Python/C API routines

- - One particular use of the cdef extern from statement is for gaining - access to routines in the Python/C API. For example, -
cdef extern from "Python.h":
- 
    object PyString_FromStringAndSize(char *s, int len)
-
- - will allow you to create Python strings containing null bytes. -

- - -
-

Resolving naming conflicts - C name specifications

- - Each Cython module has a single module-level namespace for both Python -and C names. This can be inconvenient if you want to wrap some external -C functions and provide the Python user with Python functions of the same -names. -

Cython 0.8 provides a couple of different ways of solving this problem. - The best way, especially if you have many C functions to wrap, is probably - to put the extern C function declarations into a different namespace using - the facilities described in the section on sharing - declarations between Cython modules.

- - -

The other way is to use a c name specification to give different - Cython and C names to the C function. Suppose, for example, that you want -to wrap an external function called eject_tomato. If you declare -it as

- - -
cdef extern void c_eject_tomato "eject_tomato" (float speed)
-
- - then its name inside the Cython module will be c_eject_tomato, -whereas its name in C will be eject_tomato. You can then wrap it -with -
def eject_tomato(speed):
  c_eject_tomato(speed)
-
- - so that users of your module can refer to it as eject_tomato. - -

Another use for this feature is referring to external names that happen - to be Cython keywords. For example, if you want to call an external function - called print, you can rename it to something else in your Cython -module.

- - -

As well as functions, C names can be specified for variables, structs, - unions, enums, struct and union members, and enum values. For example,

- - -
cdef extern int one "ein", two "zwei"
cdef extern float three "drei"

cdef struct spam "SPAM":
  int i "eye"
- cdef enum surprise "inquisition":
-   first "alpha"
-   second "beta" = 3
- - -
-

Public Declarations

- - You can make C variables and functions defined in a Cython module accessible - to external C code (or another Cython module) using the public keyword, as follows: -
cdef public int spam # public variable declaration

cdef public void grail(int num_nuns): # public function declaration
-     ...

-
- - If there are any public declarations in a Cython module, a .h file is generated containing equivalent C declarations for inclusion in other - C code. -

Cython also generates a .pxi file containing Cython versions of the - declarations for inclusion in another Cython module using the include statement. If you use this, you - will need to arrange for the module using the declarations to be linked -against the module defining them, and for both modules to be available to -the dynamic linker at run time. I haven't tested this, so I can't say how -well it will work on the various platforms.

- - -
NOTE: If all you want to export is an extension type, there is - now a better way -- see Sharing Declarations Between - Cython Modules.
- - -

Extension Types -

- - One of the most powerful features of Cython is the ability to easily create - new built-in Python types, called extension types. This is a major - topic in itself, so there is a  separate - page devoted to it. -

Sharing Declarations Between Cython Modules -

- - Cython 0.8 introduces a substantial new set of facilities allowing a Cython - module to easily import and use C declarations and extension types from another -Cython module. You can now create a set of co-operating Cython modules just -as easily as you can create a set of co-operating Python modules. There is -a separate page devoted to this topic. -

Limitations -

- - -

Unsupported Python features

- - Cython is not quite a full superset of Python. The following restrictions - apply: -
  • Function definitions (whether using def or cdef) - cannot be nested within other function definitions.
    -
    • -  
  • Class definitions can only appear at the top level of a module, - not inside a function.
    -
    • -  
  • The import * form of import is not allowed anywhere - (other forms of the import statement are fine, though).
    -
    • -  
  • Generators cannot be defined in Cython.
    -
    -
    • -
  • The globals() and locals() functions cannot be -used.
    • -
    - - The above restrictions will most likely remain, since removing them would - be difficult and they're not really needed for Cython's intended applications. - -

    There are also some temporary limitations, which may eventually be lifted, including: -

    - - -
  • Class and function definitions cannot be placed inside -control structures.
    -
    • -  
  • In-place arithmetic operators (+=, etc) are not yet supported.
    -
    • -  
  • List comprehensions are not yet supported.
    -
    • -  
  • There is no support for Unicode.
    -
    • -  
  • Special methods of extension types cannot have functioning -docstrings.
    -
    -
    • -
  • The use of string literals as comments is not recommended at present, - because Cython doesn't optimize them away, and won't even accept them in -places where executable statements are not allowed.
    • -
    - -

    Semantic differences between Python - and Cython

    - - -

    Behaviour of class scopes

    - - In Python, referring to a method of a class inside the class definition, - i.e. while the class is being defined, yields a plain function object, but - in Cython it yields an unbound method2. A consequence of this is that the -usual idiom for using the classmethod and staticmethod functions, e.g. -
    class Spam:
    - 
      def method(cls):
        ...
    - 
      method = classmethod(method)
    -
    - - will not work in Cython. This can be worked around by defining the function - outside the class, and then assigning the result of classmethod or - staticmethod inside the class, i.e. -
    def Spam_method(cls):
      ...
    - 
    class Spam:
    - 
      method = classmethod(Spam_method)
    -
    - - -

    Footnotes

    - - 1. A problem with const could arise if you have -something like -
    cdef extern from "grail.h":
      char *nun
    -
    - - where grail.h actually contains -
    extern const char *nun;
    -
    - - and you do -
    cdef void languissement(char *s):
      #something that doesn't change s
    - 
    ...
    - 
    languissement(nun)
    -
    - - which will cause the C compiler to complain. You can work around it by -casting away the constness: -
    languissement(<char *>nun)
    -
    - - -
    2. The reason for the different behaviour -of class scopes is that Cython-defined Python functions are PyCFunction objects, -not PyFunction objects, and are not recognised by the machinery that creates -a bound or unbound method when a function is extracted from a class. To get -around this, Cython wraps each method in an unbound method object itself before -storing it in the class's dictionary.
    - -  
    - -
    - - \ No newline at end of file diff -Nru cython-0.20.1+1~201611251650-6686/Doc/primes.c cython-0.20.1+1~202203241016-9537/Doc/primes.c --- cython-0.20.1+1~201611251650-6686/Doc/primes.c 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/primes.c 1970-01-01 00:00:00.000000000 +0000 @@ -1,142 +0,0 @@ -#include "Python.h" - - -static PyObject *__Pyx_UnpackItem(PyObject *, int); -static int __Pyx_EndUnpack(PyObject *, int); -static int __Pyx_PrintItem(PyObject *); -static int __Pyx_PrintNewline(void); -static void __Pyx_ReRaise(void); -static void __Pyx_RaiseWithTraceback(PyObject *, PyObject *, PyObject *); -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list); -static PyObject *__Pyx_GetExcValue(void); -static PyObject *__Pyx_GetName(PyObject *dict, char *name); - -static PyObject *__pyx_m; -static PyObject *__pyx_d; -static PyObject *__pyx_b; - - -PyObject *__pyx_f_primes(PyObject *__pyx_self, PyObject *__pyx_args); /*proto*/ -PyObject *__pyx_f_primes(PyObject *__pyx_self, PyObject *__pyx_args) { - int __pyx_v_kmax; - int __pyx_v_n; - int __pyx_v_k; - int __pyx_v_i; - int (__pyx_v_p[1000]); - PyObject *__pyx_v_result; - PyObject *__pyx_r; - PyObject *__pyx_1 = 0; - int __pyx_2; - int __pyx_3; - int __pyx_4; - PyObject *__pyx_5 = 0; - PyObject *__pyx_6 = 0; - if (!PyArg_ParseTuple(__pyx_args, "i", &__pyx_v_kmax)) return 0; - __pyx_v_result = Py_None; Py_INCREF(__pyx_v_result); - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":2 */ - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":3 */ - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":4 */ - __pyx_1 = PyList_New(0); if (!__pyx_1) goto __pyx_L1; - Py_DECREF(__pyx_v_result); - __pyx_v_result = __pyx_1; - __pyx_1 = 0; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":5 */ - __pyx_2 = (__pyx_v_kmax > 1000); - if (__pyx_2) { - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":6 */ - __pyx_v_kmax = 1000; - goto __pyx_L2; - } - __pyx_L2:; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":7 */ - __pyx_v_k = 0; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":8 */ - __pyx_v_n = 2; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":9 */ - while (1) { - __pyx_L3:; - __pyx_2 = (__pyx_v_k < __pyx_v_kmax); - if (!__pyx_2) break; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":10 */ - __pyx_v_i = 0; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":11 */ - while (1) { - __pyx_L5:; - if (__pyx_3 = (__pyx_v_i < __pyx_v_k)) { - __pyx_3 = ((__pyx_v_n % (__pyx_v_p[__pyx_v_i])) != 0); - } - if (!__pyx_3) break; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":12 */ - __pyx_v_i = (__pyx_v_i + 1); - } - __pyx_L6:; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":13 */ - __pyx_4 = (__pyx_v_i == __pyx_v_k); - if (__pyx_4) { - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":14 */ - (__pyx_v_p[__pyx_v_k]) = __pyx_v_n; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":15 */ - __pyx_v_k = (__pyx_v_k + 1); - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":16 */ - __pyx_1 = PyObject_GetAttrString(__pyx_v_result, "append"); if (!__pyx_1) goto __pyx_L1; - __pyx_5 = PyInt_FromLong(__pyx_v_n); if (!__pyx_5) goto __pyx_L1; - __pyx_6 = PyTuple_New(1); if (!__pyx_6) goto __pyx_L1; - PyTuple_SET_ITEM(__pyx_6, 0, __pyx_5); - __pyx_5 = 0; - __pyx_5 = PyObject_CallObject(__pyx_1, __pyx_6); if (!__pyx_5) goto __pyx_L1; - Py_DECREF(__pyx_6); __pyx_6 = 0; - Py_DECREF(__pyx_5); __pyx_5 = 0; - goto __pyx_L7; - } - __pyx_L7:; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":17 */ - __pyx_v_n = (__pyx_v_n + 1); - } - __pyx_L4:; - - /* "ProjectsA:Python:Pyrex:Demos:primes.pyx":18 */ - Py_INCREF(__pyx_v_result); - __pyx_r = __pyx_v_result; - goto __pyx_L0; - - __pyx_r = Py_None; Py_INCREF(__pyx_r); - goto __pyx_L0; - __pyx_L1:; - Py_XDECREF(__pyx_1); - Py_XDECREF(__pyx_5); - Py_XDECREF(__pyx_6); - __pyx_r = 0; - __pyx_L0:; - Py_DECREF(__pyx_v_result); - return __pyx_r; -} - -static struct PyMethodDef __pyx_methods[] = { - {"primes", (PyCFunction)__pyx_f_primes, METH_VARARGS, 0}, - {0, 0, 0, 0} -}; - -void initprimes(void); /*proto*/ -void initprimes(void) { - __pyx_m = Py_InitModule4("primes", __pyx_methods, 0, 0, PYTHON_API_VERSION); - __pyx_d = PyModule_GetDict(__pyx_m); - __pyx_b = PyImport_AddModule("__builtin__"); - PyDict_SetItemString(__pyx_d, "__builtins__", __pyx_b); -} -/* Runtime support code */ diff -Nru cython-0.20.1+1~201611251650-6686/Doc/s5/cython-ep2008.txt cython-0.20.1+1~202203241016-9537/Doc/s5/cython-ep2008.txt --- cython-0.20.1+1~201611251650-6686/Doc/s5/cython-ep2008.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/s5/cython-ep2008.txt 2022-03-24 10:16:46.000000000 +0000 @@ -51,7 +51,7 @@ * an Open-Source project - * http://cython.org + * https://cython.org/ * a Python compiler (almost) @@ -115,7 +115,7 @@ * many, *many* others - see - * http://cython.org/ + * https://cython.org/ * the mailing list archives of Cython and Pyrex @@ -398,4 +398,4 @@ \... use it, and join the project! - http://cython.org/ + https://cython.org/ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/Doc/s5/ui/default/cython-logo64.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/Doc/s5/ui/default/cython-logo64.png differ diff -Nru cython-0.20.1+1~201611251650-6686/Doc/s5/ui/default/framing.css cython-0.20.1+1~202203241016-9537/Doc/s5/ui/default/framing.css --- cython-0.20.1+1~201611251650-6686/Doc/s5/ui/default/framing.css 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/s5/ui/default/framing.css 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ /* The following styles size, place, and layer the slide components. Edit these if you want to change the overall slide layout. - The commented lines can be uncommented (and modified, if necessary) + The commented lines can be uncommented (and modified, if necessary) to help you with the rearrangement process. */ /* target = 1024x768 */ diff -Nru cython-0.20.1+1~201611251650-6686/Doc/s5/ui/default/iepngfix.htc cython-0.20.1+1~202203241016-9537/Doc/s5/ui/default/iepngfix.htc --- cython-0.20.1+1~201611251650-6686/Doc/s5/ui/default/iepngfix.htc 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Doc/s5/ui/default/iepngfix.htc 2022-03-24 10:16:46.000000000 +0000 @@ -3,7 +3,7 @@ \n", + "\n", + "\n", + "

    Generated by Cython 0.25.2

    \n", + "

    \n", + " Yellow lines hint at Python interaction.
    \n", + " Click on a line that starts with a \"+\" to see the C code that Cython generated for it.\n", + "

    \n", + "
     1:
    \n", + "
    +2: cdef int a = 0
    \n", + "
      __pyx_v_46_cython_magic_6ba45f17d130910db2606828f4326b2d_a = 0;\n",
+       "
    +3: for i in range(10):
    \n", + "
      __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_range, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "  __Pyx_GOTREF(__pyx_t_1);\n",
+       "  if (likely(PyList_CheckExact(__pyx_t_1)) || PyTuple_CheckExact(__pyx_t_1)) {\n",
+       "    __pyx_t_2 = __pyx_t_1; __Pyx_INCREF(__pyx_t_2); __pyx_t_3 = 0;\n",
+       "    __pyx_t_4 = NULL;\n",
+       "  } else {\n",
+       "    __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "    __Pyx_GOTREF(__pyx_t_2);\n",
+       "    __pyx_t_4 = Py_TYPE(__pyx_t_2)->tp_iternext; if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "  }\n",
+       "  __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;\n",
+       "  for (;;) {\n",
+       "    if (likely(!__pyx_t_4)) {\n",
+       "      if (likely(PyList_CheckExact(__pyx_t_2))) {\n",
+       "        if (__pyx_t_3 >= PyList_GET_SIZE(__pyx_t_2)) break;\n",
+       "        #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS\n",
+       "        __pyx_t_1 = PyList_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "        #else\n",
+       "        __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "        __Pyx_GOTREF(__pyx_t_1);\n",
+       "        #endif\n",
+       "      } else {\n",
+       "        if (__pyx_t_3 >= PyTuple_GET_SIZE(__pyx_t_2)) break;\n",
+       "        #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS\n",
+       "        __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3); __Pyx_INCREF(__pyx_t_1); __pyx_t_3++; if (unlikely(0 < 0)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "        #else\n",
+       "        __pyx_t_1 = PySequence_ITEM(__pyx_t_2, __pyx_t_3); __pyx_t_3++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "        __Pyx_GOTREF(__pyx_t_1);\n",
+       "        #endif\n",
+       "      }\n",
+       "    } else {\n",
+       "      __pyx_t_1 = __pyx_t_4(__pyx_t_2);\n",
+       "      if (unlikely(!__pyx_t_1)) {\n",
+       "        PyObject* exc_type = PyErr_Occurred();\n",
+       "        if (exc_type) {\n",
+       "          if (likely(exc_type == PyExc_StopIteration || PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) PyErr_Clear();\n",
+       "          else __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "        }\n",
+       "        break;\n",
+       "      }\n",
+       "      __Pyx_GOTREF(__pyx_t_1);\n",
+       "    }\n",
+       "    if (PyDict_SetItem(__pyx_d, __pyx_n_s_i, __pyx_t_1) < 0) __PYX_ERR(0, 3, __pyx_L1_error)\n",
+       "    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;\n",
+       "/* … */\n",
+       "  }\n",
+       "  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;\n",
+       "
    +4:     a += i
    \n", + "
        __pyx_t_1 = __Pyx_PyInt_From_int(__pyx_v_46_cython_magic_6ba45f17d130910db2606828f4326b2d_a); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 4, __pyx_L1_error)\n",
+       "    __Pyx_GOTREF(__pyx_t_1);\n",
+       "    __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_i); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 4, __pyx_L1_error)\n",
+       "    __Pyx_GOTREF(__pyx_t_5);\n",
+       "    __pyx_t_6 = PyNumber_InPlaceAdd(__pyx_t_1, __pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 4, __pyx_L1_error)\n",
+       "    __Pyx_GOTREF(__pyx_t_6);\n",
+       "    __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;\n",
+       "    __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0;\n",
+       "    __pyx_t_7 = __Pyx_PyInt_As_int(__pyx_t_6); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 4, __pyx_L1_error)\n",
+       "    __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;\n",
+       "    __pyx_v_46_cython_magic_6ba45f17d130910db2606828f4326b2d_a = __pyx_t_7;\n",
+       "
    +5: print(a)
    \n", + "
      __pyx_t_2 = __Pyx_PyInt_From_int(__pyx_v_46_cython_magic_6ba45f17d130910db2606828f4326b2d_a); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error)\n",
+       "  __Pyx_GOTREF(__pyx_t_2);\n",
+       "  __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 5, __pyx_L1_error)\n",
+       "  __Pyx_GOTREF(__pyx_t_6);\n",
+       "  __Pyx_GIVEREF(__pyx_t_2);\n",
+       "  PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_2);\n",
+       "  __pyx_t_2 = 0;\n",
+       "  __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_print, __pyx_t_6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 5, __pyx_L1_error)\n",
+       "  __Pyx_GOTREF(__pyx_t_2);\n",
+       "  __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;\n",
+       "  __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;\n",
+       "
    " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%cython --annotate\n", + "\n", + "cdef int a = 0\n", + "for i in range(10):\n", + " a += i\n", + "print(a)" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python [conda env:py3]", + "language": "python", + "name": "conda-env-py3-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/quickstart/cythonize.rst cython-0.20.1+1~202203241016-9537/docs/src/quickstart/cythonize.rst --- cython-0.20.1+1~201611251650-6686/docs/src/quickstart/cythonize.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/quickstart/cythonize.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,13 +1,16 @@ Faster code via static typing ============================= +.. include:: + ../two-syntax-variants-used + Cython is a Python compiler. This means that it can compile normal Python code without changes (with a few obvious exceptions of some as-yet -unsupported language features). However, for performance critical -code, it is often helpful to add static type declarations, as they -will allow Cython to step out of the dynamic nature of the Python code -and generate simpler and faster C code - sometimes faster by orders of -magnitude. +unsupported language features, see :ref:`Cython limitations`). +However, for performance critical code, it is often helpful to add +static type declarations, as they will allow Cython to step out of the +dynamic nature of the Python code and generate simpler and faster C code +- sometimes faster by orders of magnitude. It must be noted, however, that type declarations can make the source code more verbose and thus less readable. It is therefore discouraged @@ -30,35 +33,28 @@ Typing Variables ---------------- -Consider the following pure Python code:: - - def f(x): - return x**2-x +Consider the following pure Python code: - def integrate_f(a, b, N): - s = 0 - dx = (b-a)/N - for i in range(N): - s += f(a+i*dx) - return s * dx +.. literalinclude:: ../../examples/quickstart/cythonize/integrate.py + :caption: integrate.py Simply compiling this in Cython merely gives a 35% speedup. This is better than nothing, but adding some static types can make a much larger difference. -With additional type declarations, this might look like:: +With additional type declarations, this might look like: + +.. tabs:: + + .. group-tab:: Pure Python - def f(double x): - return x**2-x + .. literalinclude:: ../../examples/quickstart/cythonize/integrate_cy.py + :caption: integrate_cy.py - def integrate_f(double a, double b, int N): - cdef int i - cdef double s, dx - s = 0 - dx = (b-a)/N - for i in range(N): - s += f(a+i*dx) - return s * dx + .. group-tab:: Cython + + .. literalinclude:: ../../examples/quickstart/cythonize/integrate_cy.pyx + :caption: integrate_cy.pyx Since the iterator variable ``i`` is typed with C semantics, the for-loop will be compiled to pure C code. Typing ``a``, ``s`` and ``dx`` is important as they are involved @@ -73,28 +69,40 @@ Python function calls can be expensive -- in Cython doubly so because one might need to convert to and from Python objects to do the call. -In our example above, the argument is assumed to be a C double both inside f() +In our example above, the argument is assumed to be a C double both inside ``f()`` and in the call to it, yet a Python ``float`` object must be constructed around the argument in order to pass it. -Therefore Cython provides a syntax for declaring a C-style function, -the cdef keyword:: +Therefore, Cython provides a way for declaring a C-style function, +the Cython specific ``cdef`` statement, as well as the ``@cfunc`` decorator to +declare C-style functions in Python syntax. Both approaches are +equivalent and produce the same C code: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/quickstart/cythonize/cdef_keyword.py - cdef double f(double x) except? -2: - return x**2-x + .. group-tab:: Cython + + .. literalinclude:: ../../examples/quickstart/cythonize/cdef_keyword.pyx Some form of except-modifier should usually be added, otherwise Cython will not be able to propagate exceptions raised in the function (or a function it calls). The ``except? -2`` means that an error will be checked for if ``-2`` is returned (though the ``?`` indicates that ``-2`` may also -be used as a valid return value). +be used as a valid return value). The same can be expressed using only Python +syntax with the decorator ``@exceptval(-2, check=True)``. + Alternatively, the slower ``except *`` is always safe. An except clause can be left out if the function returns a Python object or if it is guaranteed that an exception will not be raised -within the function call. +within the function call. Again, Cython provides the decorator ``@exceptval(check=True)`` +providing the same functionality. -A side-effect of cdef is that the function is no longer available from -Python-space, as Python wouldn't know how to call it. It is also no +A side-effect of ``cdef`` (and the ``@cfunc`` decorator) is that the function is no longer +visible from Python-space, as Python wouldn't know how to call it. It is also no longer possible to change :func:`f` at runtime. Using the ``cpdef`` keyword instead of ``cdef``, a Python wrapper is also @@ -103,10 +111,13 @@ objects). In fact, ``cpdef`` does not just provide a Python wrapper, it also installs logic to allow the method to be overridden by python methods, even when called from within cython. This does add a tiny overhead compared to ``cdef`` -methods. +methods. Again, Cython provides a ``@ccall`` decorator which provides the same +functionality as ``cpdef`` keyword. Speedup: 150 times over pure Python. +.. _determining_where_to_add_types: + Determining where to add types ------------------------------ @@ -135,7 +146,15 @@ and for determining when to :ref:`release the GIL `: in general, a ``nogil`` block may contain only "white" code. -.. figure:: htmlreport.png +.. tabs:: + + .. group-tab:: Pure Python + + .. figure:: htmlreport_py.png + + .. group-tab:: Cython + + .. figure:: htmlreport_pyx.png Note that Cython deduces the type of local variables based on their assignments (including as loop variable targets) which can also cut down on the need to @@ -146,4 +165,9 @@ *integer types used in arithmetic expressions*, as Cython is unable to ensure that an overflow would not occur (and so falls back to ``object`` in case Python's bignums are needed). To allow inference of C integer types, set the -``infer_types`` :ref:`directive ` to ``True``. +``infer_types`` :ref:`directive ` to ``True``. This directive +does a work similar to the ``auto`` keyword in C++ for the readers who are familiar +with this language feature. It can be of great help to cut down on the need to type +everything, but it also can lead to surprises. Especially if one isn't familiar with +arithmetic expressions with c types. A quick overview of those +can be found `here `_. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/quickstart/demo.pyx cython-0.20.1+1~202203241016-9537/docs/src/quickstart/demo.pyx --- cython-0.20.1+1~201611251650-6686/docs/src/quickstart/demo.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/quickstart/demo.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -77,5 +77,5 @@ for i in range(N): s += f2(a+i*dx) return s * dx - + timeit(integrate_f2, "Typed func") Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/quickstart/htmlreport.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/quickstart/htmlreport.png differ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/quickstart/htmlreport_py.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/quickstart/htmlreport_py.png differ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/quickstart/htmlreport_pyx.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/quickstart/htmlreport_pyx.png differ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/quickstart/install.rst cython-0.20.1+1~202203241016-9537/docs/src/quickstart/install.rst --- cython-0.20.1+1~201611251650-6686/docs/src/quickstart/install.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/quickstart/install.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,13 +1,13 @@ +.. _install: + Installing Cython ================= Many scientific Python distributions, such as Anaconda [Anaconda]_, -Enthought Canopy [Canopy]_, Python(x,y) [Pythonxy]_, and Sage [Sage]_, +Enthought Canopy [Canopy]_, and Sage [Sage]_, bundle Cython and no setup is needed. Note however that if your distribution ships a version of Cython which is too old you can still -use the instructions below to update Cython. Everything in this -tutorial should work with Cython 0.11.2 and newer, unless a footnote -says otherwise. +use the instructions below to update Cython. Unlike most Python software, Cython requires a C compiler to be present on the system. The details of getting a C compiler varies @@ -20,32 +20,33 @@ - **Mac OS X** To retrieve gcc, one option is to install Apple's XCode, which can be retrieved from the Mac OS X's install DVDs or - from http://developer.apple.com. + from https://developer.apple.com/. - - **Windows** A popular option is to use the open source MinGW (a + - **Windows** The CPython project recommends building extension modules + (including Cython modules) with the same compiler that Python was + built with. This is usually a specific version of Microsoft Visual + C/C++ (MSVC) - see https://wiki.python.org/moin/WindowsCompilers. + MSVC is the only compiler that Cython is currently tested with on + Windows. A possible alternative is the open source MinGW (a Windows distribution of gcc). See the appendix for instructions for setting up MinGW manually. Enthought Canopy and Python(x,y) bundle MinGW, but some of the configuration steps in the appendix might - still be necessary. Another option is to use Microsoft's Visual C. - One must then use the same version which the installed Python was - compiled with. + still be necessary. .. dagss tried other forms of ReST lists and they didn't look nice .. with rst2latex. +The simplest way of installing Cython is by using ``pip``:: + + pip install Cython + + The newest Cython release can always be downloaded from -http://cython.org. Unpack the tarball or zip file, enter the +https://cython.org/. Unpack the tarball or zip file, enter the directory, and then run:: python setup.py install -If you have ``pip`` set up on your system (e.g. in a virtualenv or a -recent Python version), you should be able to fetch Cython from PyPI -and install it using - -:: - - pip install Cython For one-time builds, e.g. for CI/testing, on platforms that are not covered by one of the wheel packages provided on PyPI, it is substantially faster @@ -57,7 +58,6 @@ pip install Cython --install-option="--no-cython-compile" -.. [Anaconda] http://docs.continuum.io/anaconda/ -.. [Canopy] https://enthought.com/products/canopy/ -.. [Pythonxy] http://www.pythonxy.com/ -.. [Sage] W. Stein et al., Sage Mathematics Software, http://sagemath.org +.. [Anaconda] https://docs.anaconda.com/anaconda/ +.. [Canopy] https://www.enthought.com/product/canopy/ +.. [Sage] W. Stein et al., Sage Mathematics Software, https://www.sagemath.org/ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/quickstart/ipython.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/quickstart/ipython.png differ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/quickstart/jupyter.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/quickstart/jupyter.png differ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/quickstart/overview.rst cython-0.20.1+1~202203241016-9537/docs/src/quickstart/overview.rst --- cython-0.20.1+1~201611251650-6686/docs/src/quickstart/overview.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/quickstart/overview.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,7 @@ Cython - an overview ==================== -[Cython] is a programming language that makes writing C extensions +[Cython]_ is a programming language that makes writing C extensions for the Python language as easy as Python itself. It aims to become a superset of the [Python]_ language which gives it high-level, object-oriented, functional, and dynamic programming. Its main feature @@ -25,7 +25,7 @@ has approached this problem by means of a source code compiler that translates Python code to equivalent C code. This code is executed within the CPython runtime environment, but at the speed of compiled C -and with the ability to call directly into C libraries. +and with the ability to call directly into C libraries. At the same time, it keeps the original interface of the Python source code, which makes it directly usable from Python code. These two-fold characteristics enable Cython's two major use cases: @@ -44,13 +44,13 @@ language. .. [Cython] G. Ewing, R. W. Bradshaw, S. Behnel, D. S. Seljebotn et al., - The Cython compiler, http://cython.org. -.. [IronPython] Jim Hugunin et al., http://www.codeplex.com/IronPython. + The Cython compiler, https://cython.org/. +.. [IronPython] Jim Hugunin et al., https://archive.codeplex.com/?p=IronPython. .. [Jython] J. Huginin, B. Warsaw, F. Bock, et al., - Jython: Python for the Java platform, http://www.jython.org. -.. [PyPy] The PyPy Group, PyPy: a Python implementation written in Python, - http://pypy.org. + Jython: Python for the Java platform, https://www.jython.org. +.. [PyPy] The PyPy Group, PyPy: a Python implementation written in Python, + https://pypy.org/. .. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [Python] G. van Rossum et al., The Python programming language, - http://python.org. + https://www.python.org/. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/compilation.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/compilation.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/compilation.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/compilation.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,514 +1,103 @@ .. highlight:: cython -.. _compilation-reference: - ============= Compilation ============= -Cython code, unlike Python, must be compiled. This happens in two stages: - - * A ``.pyx`` file is compiled by Cython to a ``.c`` file. +.. note:: - * The ``.c`` file is compiled by a C compiler to a ``.so`` file (or a - ``.pyd`` file on Windows) + The sections in this page were moved to the :ref:`compilation` in the userguide. -The following sub-sections describe several ways to build your -extension modules, and how to pass directives to the Cython compiler. - Compiling from the command line =============================== -Run the Cython compiler command with your options and list of ``.pyx`` -files to generate. For example:: - - $ cython -a yourmod.pyx - -This creates a ``yourmod.c`` file, and the ``-a`` switch produces an -annotated html file of the source code. Pass the ``-h`` flag for a -complete list of supported flags. - -Compiling your ``.c`` files will vary depending on your operating -system. Python documentation for writing extension modules should -have some details for your system. Here we give an example on a Linux -system:: - - $ gcc -shared -pthread -fPIC -fwrapv -O2 -Wall -fno-strict-aliasing \ - -I/usr/include/python2.7 -o yourmod.so yourmod.c - -[``gcc`` will need to have paths to your included header files and -paths to libraries you need to link with] - -A ``yourmod.so`` file is now in the same directory and your module, -``yourmod``, is available for you to import as you normally would. - +This section was moved to :ref:`compiling_command_line`. Compiling with ``distutils`` ============================ -The ``distutils`` package is part of the standard library. It is the standard -way of building Python packages, including native extension modules. The -following example configures the build for a Cython file called *hello.pyx*. -First, create a ``setup.py`` script:: - - from distutils.core import setup - from Cython.Build import cythonize - - setup( - name = "My hello app", - ext_modules = cythonize('hello.pyx'), # accepts a glob pattern - ) - -Now, run the command ``python setup.py build_ext --inplace`` in your -system's command shell and you are done. Import your new extension -module into your python shell or script as normal. - -The ``cythonize`` command also allows for multi-threaded compilation and -dependency resolution. Recompilation will be skipped if the target file -is up to date with its main source file and dependencies. - +This section was moved to :ref:`basic_setup.py`. Configuring the C-Build ------------------------ -If you have include files in non-standard places you can pass an -``include_path`` parameter to ``cythonize``:: - - from distutils.core import setup - from Cython.Build import cythonize - - setup( - name = "My hello app", - ext_modules = cythonize("src/*.pyx", include_path = [...]), - ) +This section was moved to :ref:`basic_setup.py`. -Often, Python packages that offer a C-level API provide a way to find -the necessary include files, e.g. for NumPy:: +Cythonize arguments +------------------- - include_path = [numpy.get_include()] +This section was moved to :ref:`cythonize_arguments`. -Note for Numpy users. Despite this, you will still get warnings like the -following from the compiler, because Cython is using a deprecated Numpy API:: +Compiler options +---------------- - .../include/numpy/npy_1_7_deprecated_api.h:15:2: warning: #warning "Using deprecated NumPy API, disable it by " "#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-Wcpp] +This section was moved to :ref:`compiler_options`. -For the time being, it is just a warning that you can ignore. - -If you need to specify compiler options, libraries to link with or other -linker options you will need to create ``Extension`` instances manually -(note that glob syntax can still be used to specify multiple extensions -in one line):: - - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize - - extensions = [ - Extension("primes", ["primes.pyx"], - include_dirs = [...], - libraries = [...], - library_dirs = [...]), - # Everything but primes.pyx is included here. - Extension("*", ["*.pyx"], - include_dirs = [...], - libraries = [...], - library_dirs = [...]), - ] - setup( - name = "My hello app", - ext_modules = cythonize(extensions), - ) - -If your options are static (for example you do not need to call a tool like -``pkg-config`` to determine them) you can also provide them directly in your -.pyx source file using a special comment block at the start of the file:: - - # distutils: libraries = spam eggs - # distutils: include_dirs = /opt/food/include - -If you have some C files that have been wrapped with Cython and you want to -compile them into your extension, you can define the distutils ``sources`` -parameter:: - - # distutils: sources = helper.c, another_helper.c - -Note that these sources are added to the list of sources of the current -extension module. Spelling this out in the :file:`setup.py` file looks -as follows:: - - from distutils.core import setup - from Cython.Build import cythonize - from distutils.extension import Extension - - sourcefiles = ['example.pyx', 'helper.c', 'another_helper.c'] +Distributing Cython modules +---------------------------- - extensions = [Extension("example", sourcefiles)] +This section was moved to :ref:`distributing_cython_modules`. - setup( - ext_modules = cythonize(extensions) - ) +Integrating multiple modules +============================ -The :class:`Extension` class takes many options, and a fuller explanation can -be found in the `distutils documentation`_. Some useful options to know about -are ``include_dirs``, ``libraries``, and ``library_dirs`` which specify where -to find the ``.h`` and library files when linking to external libraries. +This section was moved to :ref:`integrating_multiple_modules`. -.. _distutils documentation: http://docs.python.org/extending/building.html +Compiling with :mod:`pyximport` +=============================== +This section was moved to :ref:`pyximport`. -Distributing Cython modules ----------------------------- +Arguments +--------- -It is strongly recommended that you distribute the generated ``.c`` files as well -as your Cython sources, so that users can install your module without needing -to have Cython available. - -It is also recommended that Cython compilation not be enabled by default in the -version you distribute. Even if the user has Cython installed, he/she probably -doesn't want to use it just to install your module. Also, the installed version -may not be the same one you used, and may not compile your sources correctly. - -This simply means that the :file:`setup.py` file that you ship with will just -be a normal distutils file on the generated `.c` files, for the basic example -we would have instead:: - - from distutils.core import setup - from distutils.extension import Extension - - setup( - ext_modules = [Extension("example", ["example.c"])] - ) - -This is easy to combine with :func:`cythonize` by changing the file extension -of the extension module sources:: - - from distutils.core import setup - from distutils.extension import Extension - - USE_CYTHON = ... # command line option, try-import, ... - - ext = '.pyx' if USE_CYTHON else '.c' - - extensions = [Extension("example", ["example"+ext])] - - if USE_CYTHON: - from Cython.Build import cythonize - extensions = cythonize(extensions) - - setup( - ext_modules = extensions - ) - -If you have many extensions and want to avoid the additional complexity in the -declarations, you can declare them with their normal Cython sources and then -call the following function instead of ``cythonize()`` to adapt the sources -list in the Extensions when not using Cython:: - - import os.path - - def no_cythonize(extensions, **_ignore): - for extension in extensions: - sources = [] - for sfile in extension.sources: - path, ext = os.path.splitext(sfile) - if ext in ('.pyx', '.py'): - if extension.language == 'c++': - ext = '.cpp' - else: - ext = '.c' - sfile = path + ext - sources.append(sfile) - extension.sources[:] = sources - return extensions - -Another option is to make Cython a setup dependency of your system and use -Cython's build_ext module which runs ``cythonize`` as part of the build process:: - - setup( - setup_requires=[ - 'cython>=0.x', - ], - extensions = [Extension("*", ["*.pyx"])], - cmdclass={'build_ext': Cython.Build.build_ext}, - ... - ) - -If you want to expose the C-level interface of your library for other -libraries to cimport from, use package_data to install the ``.pxd`` files, -e.g.:: - - setup( - package_data = { - 'my_package': ['*.pxd'], - 'my_package/sub_package': ['*.pxd'], - }, - ... - ) - -These ``.pxd`` files need not correspond have corresponding ``.pyx`` -modules if they contain purely declarations of external libraries. - -Compiling with ``pyximport`` -============================= - -For generating Cython code right in your pure python module just type:: - - >>> import pyximport; pyximport.install() - >>> import helloworld - Hello World - -This allows you to automatically run Cython on every ``.pyx`` that -Python is trying to import. You should use this for simple Cython -builds only where no extra C libraries and no special building setup -is needed. - -In the case that Cython fails to compile a Python module, *pyximport* -will fall back to loading the source modules instead. - -It is also possible to compile new ``.py`` modules that are being -imported (including the standard library and installed packages). For -using this feature, just tell that to ``pyximport``:: +Dependency Handling +-------------------- - >>> pyximport.install(pyimport = True) +Limitations +------------ Compiling with ``cython.inline`` ================================= -One can also compile Cython in a fashion similar to SciPy's ``weave.inline``. -For example:: - - >>> import cython - >>> def f(a): - ... ret = cython.inline("return a+b", b=3) - ... - -Unbound variables are automatically pulled from the surrounding local -and global scopes, and the result of the compilation is cached for -efficient re-use. +This section was moved to :ref:`compiling_with_cython_inline`. Compiling with Sage =================== -The Sage notebook allows transparently editing and compiling Cython -code simply by typing ``%cython`` at the top of a cell and evaluate -it. Variables and functions defined in a Cython cell are imported into the -running session. Please check `Sage documentation -`_ for details. +This section was moved to :ref:`compiling_with_sage`. -You can tailor the behavior of the Cython compiler by specifying the -directives below. +Compiling with a Jupyter Notebook +================================= -.. _compiler-directives: +This section was moved to :ref:`compiling_notebook`. Compiler directives ==================== -Compiler directives are instructions which affect the behavior of -Cython code. Here is the list of currently supported directives: - -``boundscheck`` (True / False) - If set to False, Cython is free to assume that indexing operations - ([]-operator) in the code will not cause any IndexErrors to be - raised. Lists, tuples, and strings are affected only if the index - can be determined to be non-negative (or if ``wraparound`` is False). - Conditions - which would normally trigger an IndexError may instead cause - segfaults or data corruption if this is set to False. - Default is True. - -``wraparound`` (True / False) - In Python arrays can be indexed relative to the end. For example - A[-1] indexes the last value of a list. In C negative indexing is - not supported. If set to False, Cython will neither check for nor - correctly handle negative indices, possibly causing segfaults or - data corruption. - Default is True. - -``initializedcheck`` (True / False) - If set to True, Cython checks that a memoryview is initialized - whenever its elements are accessed or assigned to. Setting this - to False disables these checks. - Default is True. - -``nonecheck`` (True / False) - If set to False, Cython is free to assume that native field - accesses on variables typed as an extension type, or buffer - accesses on a buffer variable, never occurs when the variable is - set to ``None``. Otherwise a check is inserted and the - appropriate exception is raised. This is off by default for - performance reasons. Default is False. - -``overflowcheck`` (True / False) - If set to True, raise errors on overflowing C integer arithmetic - operations. Incurs a modest runtime penalty, but is much faster than - using Python ints. Default is False. - -``overflowcheck.fold`` (True / False) - If set to True, and overflowcheck is True, check the overflow bit for - nested, side-effect-free arithmetic expressions once rather than at every - step. Depending on the compiler, architecture, and optimization settings, - this may help or hurt performance. A simple suite of benchmarks can be - found in ``Demos/overflow_perf.pyx``. Default is True. - -``embedsignature`` (True / False) - If set to True, Cython will embed a textual copy of the call - signature in the docstring of all Python visible functions and - classes. Tools like IPython and epydoc can thus display the - signature, which cannot otherwise be retrieved after - compilation. Default is False. - -``cdivision`` (True / False) - If set to False, Cython will adjust the remainder and quotient - operators C types to match those of Python ints (which differ when - the operands have opposite signs) and raise a - ``ZeroDivisionError`` when the right operand is 0. This has up to - a 35% speed penalty. If set to True, no checks are performed. See - `CEP 516 `_. Default - is False. - -``cdivision_warnings`` (True / False) - If set to True, Cython will emit a runtime warning whenever - division is performed with negative operands. See `CEP 516 - `_. Default is - False. - -``always_allow_keywords`` (True / False) - Avoid the ``METH_NOARGS`` and ``METH_O`` when constructing - functions/methods which take zero or one arguments. Has no effect - on special methods and functions with more than one argument. The - ``METH_NOARGS`` and ``METH_O`` signatures provide faster - calling conventions but disallow the use of keywords. - -``profile`` (True / False) - Write hooks for Python profilers into the compiled C code. Default - is False. - -``linetrace`` (True / False) - Write line tracing hooks for Python profilers or coverage reporting - into the compiled C code. This also enables profiling. Default is - False. Note that the generated module will not actually use line - tracing, unless you additionally pass the C macro definition - ``CYTHON_TRACE=1`` to the C compiler (e.g. using the distutils option - ``define_macros``). Define ``CYTHON_TRACE_NOGIL=1`` to also include - ``nogil`` functions and sections. - -``infer_types`` (True / False) - Infer types of untyped variables in function bodies. Default is - None, indicating that only safe (semantically-unchanging) inferences - are allowed. - In particular, inferring *integral* types for variables *used in arithmetic - expressions* is considered unsafe (due to possible overflow) and must be - explicitly requested. - -``language_level`` (2/3) - Globally set the Python language level to be used for module - compilation. Default is compatibility with Python 2. To enable - Python 3 source code semantics, set this to 3 at the start of a - module or pass the "-3" command line option to the compiler. - Note that cimported and included source files inherit this - setting from the module being compiled, unless they explicitly - set their own language level. - -``c_string_type`` (bytes / str / unicode) - Globally set the type of an implicit coercion from char* or std::string. - -``c_string_encoding`` (ascii, default, utf-8, etc.) - Globally set the encoding to use when implicitly coercing char* or std:string - to a unicode object. Coercion from a unicode object to C type is only allowed - when set to ``ascii`` or ``default``, the latter being utf-8 in Python 3 and - nearly-always ascii in Python 2. - -``type_version_tag`` (True / False) - Enables the attribute cache for extension types in CPython by setting the - type flag ``Py_TPFLAGS_HAVE_VERSION_TAG``. Default is True, meaning that - the cache is enabled for Cython implemented types. To disable it - explicitly in the rare cases where a type needs to juggle with its ``tp_dict`` - internally without paying attention to cache consistency, this option can - be set to False. - -``unraisable_tracebacks`` (True / False) - Whether to print tracebacks when suppressing unraisable exceptions. - +This section was moved to :ref:`compiler-directives`. Configurable optimisations -------------------------- -``optimize.use_switch`` (True / False) - Whether to expand chained if-else statements (including statements like - ``if x == 1 or x == 2:``) into C switch statements. This can have performance - benefits if there are lots of values but cause compiler errors if there are any - duplicate values (which may not be detectable at Cython compile time for all - C constants). Default is True. - -``optimize.unpack_method_calls`` (True / False) - Cython can generate code that optimistically checks for Python method objects - at call time and unpacks the underlying function to call it directly. This - can substantially speed up method calls, especially for builtins, but may also - have a slight negative performance impact in some cases where the guess goes - completely wrong. - Disabling this option can also reduce the code size. Default is True. +This section was moved to :ref:`configurable_optimisations`. +Warnings +-------- + +This section was moved to :ref:`warnings`. How to set directives --------------------- +This section was moved to :ref:`how_to_set_directives`. + Globally ::::::::: -One can set compiler directives through a special header comment at the top of the file, like this:: - - #!python - #cython: language_level=3, boundscheck=False - -The comment must appear before any code (but can appear after other -comments or whitespace). - -One can also pass a directive on the command line by using the -X switch:: - - $ cython -X boundscheck=True ... - -Directives passed on the command line will override directives set in -header comments. - Locally :::::::: -For local blocks, you need to cimport the special builtin ``cython`` -module:: - - #!python - cimport cython - -Then you can use the directives either as decorators or in a with -statement, like this:: - - #!python - @cython.boundscheck(False) # turn off boundscheck for this function - def f(): - ... - # turn it temporarily on again for this block - with cython.boundscheck(True): - ... - -.. Warning:: These two methods of setting directives are **not** - affected by overriding the directive on the command-line using the - -X option. - In :file:`setup.py` ::::::::::::::::::: - -Compiler directives can also be set in the :file:`setup.py` file by passing a keyword -argument to ``cythonize``:: - - from distutils.core import setup - from Cython.Build import cythonize - - setup( - name = "My hello app", - ext_modules = cythonize('hello.pyx', compiler_directives={'embedsignature': True}), - ) - -This will override the default directives as specified in the ``compiler_directives`` dictionary. -Note that explicit per-file or local directives as explained above take precedence over the -values passed to ``cythonize``. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/directives.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/directives.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/directives.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/directives.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,6 @@ +:orphan: + Compiler Directives =================== -See `Compilation `_. +See :ref:`compiler-directives`. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/extension_types.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/extension_types.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/extension_types.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/extension_types.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,430 +1,102 @@ -.. highlight:: cython +:orphan: -.. _extension_types: +.. highlight:: cython *************** Extension Types *************** -* Normal Python as well as extension type classes can be defined. -* Extension types: - - * Are considered by Python as "built-in" types. - * Can be used to wrap arbitrary C-data structures, and provide a Python-like interface to them from Python. - * Attributes and methods can be called from Python or Cython code - * Are defined by the ``cdef class`` statement. - -:: - - cdef class Shrubbery: - - cdef int width, height - - def __init__(self, w, h): - self.width = w - self.height = h +.. note:: - def describe(self): - print "This shrubbery is", self.width, \ - "by", self.height, "cubits." + The sections in this page were moved to the :ref:`extension-types` + and :ref:`special-methods` in the userguide. ========== Attributes ========== -* Are stored directly in the object's C struct. -* Are fixed at compile time. - - * You can't add attributes to an extension type instance at run time like in normal Python, unless you define a ``__dict__`` attribute. - * You can sub-class the extension type in Python to add attributes at run-time. - -* There are two ways to access extension type attributes: - - * By Python look-up. - - * Python code's only method of access. - - * By direct access to the C struct from Cython code. - - * Cython code can use either method of access, though. - -* By default, extension type attributes are: - - * Only accessible by direct access. - * Not accessible from Python code. - -* To make attributes accessible to Python, they must be declared ``public`` or ``readonly``:: - - cdef class Shrubbery: - cdef public int width, height - cdef readonly float depth - - * The ``width`` and ``height`` attributes are readable and writable from Python code. - * The ``depth`` attribute is readable but not writable. - -.. note:: - .. note:: - You can only expose simple C types, such as ints, floats, and strings, for Python access. You can also expose Python-valued attributes. - - .. note:: - The ``public`` and ``readonly`` options apply only to Python access, not direct access. All the attributes of an extension type are always readable and writable by C-level access. - +This section was moved to :ref:`readonly`. ======= Methods ======= -* ``self`` is used in extension type methods just like it normally is in Python. -* See **Functions and Methods**; all of which applies here. - ========== Properties ========== -* Cython provides a special (deprecated) syntax:: - - cdef class Spam: - - property cheese: - - "A doc string can go here." - - def __get__(self): - # This is called when the property is read. - ... - - def __set__(self, value): - # This is called when the property is written. - ... - - def __del__(self): - # This is called when the property is deleted. - -* The ``__get__()``, ``__set__()``, and ``__del__()`` methods are all optional. - - * If they are omitted, an exception is raised on attribute access. - -* Below, is a full example that defines a property which can.. - - * Add to a list each time it is written to (``"__set__"``). - * Return the list when it is read (``"__get__"``). - * Empty the list when it is deleted (``"__del__"``). - -:: - - # cheesy.pyx - cdef class CheeseShop: - - cdef object cheeses - - def __cinit__(self): - self.cheeses = [] - - property cheese: # note that this syntax is deprecated - - def __get__(self): - return "We don't have: %s" % self.cheeses - - def __set__(self, value): - self.cheeses.append(value) - - def __del__(self): - del self.cheeses[:] - - # Test input - from cheesy import CheeseShop - - shop = CheeseShop() - print shop.cheese - - shop.cheese = "camembert" - print shop.cheese - - shop.cheese = "cheddar" - print shop.cheese - - del shop.cheese - print shop.cheese - -:: - - # Test output - We don't have: [] - We don't have: ['camembert'] - We don't have: ['camembert', 'cheddar'] - We don't have: [] - +This section was moved to :ref:`properties`. =============== Special Methods =============== -.. note:: - - #. The semantics of Cython's special methods are similar in principle to that of Python's. - #. There are substantial differences in some behavior. - #. Some Cython special methods have no Python counter-part. - -* See the :ref:`special_methods_table` for the many that are available. - +This section was moved to :ref:`special-methods`. Declaration =========== -* Must be declared with ``def`` and cannot be declared with ``cdef``. -* Performance is not affected by the ``def`` declaration because of special calling conventions +This section was moved to :ref:`declaration`. Docstrings ========== -* Docstrings are not supported yet for some special method types. -* They can be included in the source, but may not appear in the corresponding ``__doc__`` attribute at run-time. - - * This a Python library limitation because the ``PyTypeObject`` data structure is limited - +This section was moved to :ref:`docstrings`. Initialization: ``__cinit__()`` and ``__init__()`` ================================================== -* Any arguments passed to the extension type's constructor - will be passed to both initialization methods. - -* ``__cinit__()`` is where you should perform C-level initialization of the object - - * This includes any allocation of C data structures. - * **Caution** is warranted as to what you do in this method. - - * The object may not be fully valid Python object when it is called. - * Calling Python objects, including the extensions own methods, may be hazardous. - - * By the time ``__cinit__()`` is called... - - * Memory has been allocated for the object. - * All C-level attributes have been initialized to 0 or null. - * Python have been initialized to ``None``, but you can not rely on that for each occasion. - * This initialization method is guaranteed to be called exactly once. - - * For Extensions types that inherit a base type: - - * The ``__cinit__()`` method of the base type is automatically called before this one. - * The inherited ``__cinit__()`` method can not be called explicitly. - * Passing modified argument lists to the base type must be done through ``__init__()``. - * It may be wise to give the ``__cinit__()`` method both ``"*"`` and ``"**"`` arguments. - - * Allows the method to accept or ignore additional arguments. - * Eliminates the need for a Python level sub-class, that changes the ``__init__()`` - method's signature, to have to override both the ``__new__()`` and ``__init__()`` methods. - - * If ``__cinit__()`` is declared to take no arguments except ``self``, it will ignore any - extra arguments passed to the constructor without complaining about a signature mis-match. - - -* ``__init__()`` is for higher-level initialization and is safer for Python access. - - * By the time this method is called, the extension type is a fully valid Python object. - * All operations are safe. - * This method may sometimes be called more than once, or possibly not at all. - - * Take this into consideration to make sure the design of your other methods are robust of this fact. - -Note that all constructor arguments will be passed as Python objects. -This implies that non-convertible C types such as pointers or C++ objects -cannot be passed into the constructor from Cython code. If this is needed, -use a factory function instead that handles the object initialisation. -It often helps to directly call ``__new__()`` in this function to bypass the -call to the ``__init__()`` constructor. - +This section was moved to :ref:`initialisation_methods`. Finalization: ``__dealloc__()`` =============================== -* This method is the counter-part to ``__cinit__()``. -* Any C-data that was explicitly allocated in the ``__cinit__()`` method should be freed here. -* Use caution in this method: - - * The Python object to which this method belongs may not be completely intact at this point. - * Avoid invoking any Python operations that may touch the object. - * Don't call any of this object's methods. - * It's best to just deallocate C-data structures here. - -* All Python attributes of your extension type object are deallocated by Cython after the ``__dealloc__()`` method returns. +This section was moved to :ref:`finalization_method`. Arithmetic Methods ================== -.. note:: Most of these methods behave differently than in Python - -* There are not "reversed" versions of these methods... there is no __radd__() for instance. -* If the first operand cannot perform the operation, the same method of the second operand is called, with the operands in the same order. -* Do not rely on the first parameter of these methods, being ``"self"`` or the right type. -* The types of both operands should be tested before deciding what to do. -* Return ``NotImplemented`` for unhandled, mis-matched operand types. -* The previously mentioned points.. - - * Also apply to 'in-place' method ``__ipow__()``. - * Do not apply to other 'in-place' methods like ``__iadd__()``, in that these always take ``self`` as the first argument. - +This section was moved to :ref:`arithmetic_methods`. Rich Comparisons ================ -.. note:: There are no separate methods for individual rich comparison operations. - -* A single special method called ``__richcmp__()`` replaces all the individual rich compare, special method types. -* ``__richcmp__()`` takes an integer argument, indicating which operation is to be performed as shown in the table below. - - +-----+-----+ - | < | 0 | - +-----+-----+ - | == | 2 | - +-----+-----+ - | > | 4 | - +-----+-----+ - | <= | 1 | - +-----+-----+ - | != | 3 | - +-----+-----+ - | >= | 5 | - +-----+-----+ - - - +This section was moved to :ref:`rich_comparisons`. The ``__next__()`` Method ========================= -* Extension types used to expose an iterator interface should define a ``__next__()`` method. -* **Do not** explicitly supply a ``next()`` method, because Python does that for you automatically. - +This section was moved to :ref:`the__next__method`. =========== Subclassing =========== -* An extension type may inherit from a built-in type or another extension type:: - - cdef class Parrot: - ... - - cdef class Norwegian(Parrot): - ... - -* A complete definition of the base type must be available to Cython - - * If the base type is a built-in type, it must have been previously declared as an ``extern`` extension type. - * ``cimport`` can be used to import the base type, if the extern declared base type is in a ``.pxd`` definition file. - - * In Cython, multiple inheritance is not permitted.. singular inheritance only - -* Cython extension types can also be sub-classed in Python. - - * Here multiple inheritance is permissible as is normal for Python. - * Even multiple extension types may be inherited, but C-layout of all the base classes must be compatible. - +This section was moved to :ref:`subclassing`. ==================== Forward Declarations ==================== -* Extension types can be "forward-declared". -* This is necessary when two extension types refer to each other:: - - cdef class Shrubbery # forward declaration - - cdef class Shrubber: - cdef Shrubbery work_in_progress - - cdef class Shrubbery: - cdef Shrubber creator - -* An extension type that has a base-class, requires that both forward-declarations be specified:: - - cdef class A(B) - - ... - - cdef class A(B): - # attributes and methods - +This section was moved to :ref:`forward_declaring_extension_types`. ======================== Extension Types and None ======================== -* Parameters and C-variables declared as an Extension type, may take the value of ``None``. -* This is analogous to the way a C-pointer can take the value of ``NULL``. - -.. note:: - #. Exercise caution when using ``None`` - #. Read this section carefully. - -* There is no problem as long as you are performing Python operations on it. - - * This is because full dynamic type checking is applied - -* When accessing an extension type's C-attributes, **make sure** it is not ``None``. - - * Cython does not check this for reasons of efficiency. - -* Be very aware of exposing Python functions that take extension types as arguments:: - - def widen_shrubbery(Shrubbery sh, extra_width): # This is - sh.width = sh.width + extra_width - - * Users could **crash** the program by passing ``None`` for the ``sh`` parameter. - * This could be avoided by:: - - def widen_shrubbery(Shrubbery sh, extra_width): - if sh is None: - raise TypeError - sh.width = sh.width + extra_width - - * Cython provides a more convenient way with a ``not None`` clause:: - - def widen_shrubbery(Shrubbery sh not None, extra_width): - sh.width = sh.width + extra_width - - * Now this function automatically checks that ``sh`` is not ``None``, as well as that is the right type. - -* ``not None`` can only be used in Python functions (declared with ``def`` **not** ``cdef``). -* For ``cdef`` functions, you will have to provide the check yourself. -* The ``self`` parameter of an extension type is guaranteed to **never** be ``None``. -* When comparing a value ``x`` with ``None``, and ``x`` is a Python object, note the following: - - * ``x is None`` and ``x is not None`` are very efficient. - - * They translate directly to C-pointer comparisons. - - * ``x == None`` and ``x != None`` or ``if x: ...`` (a boolean condition), will invoke Python operations and will therefore be much slower. +This section was moved to :ref:`extension_types_and_none`. ================ Weak Referencing ================ -* By default, weak references are not supported. -* It can be enabled by declaring a C attribute of the ``object`` type called ``__weakref__()``:: - - cdef class ExplodingAnimal: - """This animal will self-destruct when it is - no longer strongly referenced.""" - - cdef object __weakref__ +This section was moved to :ref:`making_extension_types_weak_referenceable`. ================== Dynamic Attributes ================== -* By default, you cannot dynamically add attributes to a ``cdef class`` instance at runtime. -* It can be enabled by declaring a C attribute of the ``dict`` type called ``__dict__``:: - - cdef class ExtendableAnimal: - """This animal can be extended with new - attributes at runtime.""" - - cdef dict __dict__ - -.. note:: - #. This can have a performance penalty, especially when using ``cpdef`` methods in a class. +This section was moved to :ref:`dynamic_attributes`. ========================= External and Public Types @@ -434,118 +106,20 @@ Public ====== -* When an extension type is declared ``public``, Cython will generate a C-header (".h") file. -* The header file will contain the declarations for it's **object-struct** and it's **type-object**. -* External C-code can now access the attributes of the extension type. - +This section was moved to :ref:`public`. External ======== -* An ``extern`` extension type allows you to gain access to the internals of: - - * Python objects defined in the Python core. - * Non-Cython extension modules - -* The following example lets you get at the C-level members of Python's built-in "complex" object:: - - cdef extern from "complexobject.h": - - struct Py_complex: - double real - double imag - - ctypedef class __builtin__.complex [object PyComplexObject]: - cdef Py_complex cval - - # A function which uses the above type - def spam(complex c): - print "Real:", c.cval.real - print "Imag:", c.cval.imag - -.. note:: Some important things in the example: - #. ``ctypedef`` has been used because Python's header file has the struct declared with:: - - ctypedef struct { - ... - } PyComplexObject; - - #. The module of where this type object can be found is specified along side the name of the extension type. See **Implicit Importing**. - - #. When declaring an external extension type... - - * Don't declare any methods, because they are Python method class the are not needed. - * Similar to **structs** and **unions**, extension classes declared inside a ``cdef extern from`` block only need to declare the C members which you will actually need to access in your module. - +This section was moved to :ref:`external_extension_types`. Name Specification Clause ========================= -.. note:: Only available to **public** and **extern** extension types. - -* Example:: - - [object object_struct_name, type type_object_name ] - -* ``object_struct_name`` is the name to assume for the type's C-struct. -* ``type_object_name`` is the name to assume for the type's statically declared type-object. -* The object and type clauses can be written in any order. -* For ``cdef extern from`` declarations, This clause **is required**. - - * The object clause is required because Cython must generate code that is compatible with the declarations in the header file. - * Otherwise the object clause is optional. - -* For public extension types, both the object and type clauses **are required** for Cython to generate code that is compatible with external C-code. +This section was moved to :ref:`name_specification_clause`. ================================ Type Names vs. Constructor Names ================================ -* In a Cython module, the name of an extension type serves two distinct purposes: - - #. When used in an expression, it refers to a "module-level" global variable holding the type's constructor (i.e. it's type-object) - #. It can also be used as a C-type name to declare a "type" for variables, arguments, and return values. - -* Example:: - - cdef extern class MyModule.Spam: - ... - - * The name "Spam" serves both of these roles. - * Only "Spam" can be used as the type-name. - * The constructor can be referred to by other names. - * Upon an explicit import of "MyModule"... - - * ``MyModule.Spam()`` could be used as the constructor call. - * ``MyModule.Spam`` could not be used as a type-name - -* When an "as" clause is used, the name specified takes over both roles:: - - cdef extern class MyModule.Spam as Yummy: - ... - - * ``Yummy`` becomes both type-name and a name for the constructor. - * There other ways of course, to get hold of the constructor, but ``Yummy`` is the only usable type-name. - - - - - - - - - - - - - - - - - - - - - - - +This section was moved to :ref:`types_names_vs_constructor_names`. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/index.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/index.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/index.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/index.rst 2022-03-24 10:16:46.000000000 +0000 @@ -11,12 +11,7 @@ :maxdepth: 2 compilation - language_basics - extension_types - interfacing_with_other_code - special_mention - limitations - directives + Indices and tables ------------------ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/language_basics.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/language_basics.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/language_basics.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/language_basics.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,23 +1,21 @@ -.. highlight:: cython - +:orphan: +.. highlight:: cython -.. _language_basics: *************** Language Basics *************** +.. note:: + + The sections in this page were moved to the :ref:`language-basics` in the userguide. + ================= Cython File Types ================= -There are three file types in Cython: - -* Implementation files carry a ``.pyx`` suffix -* Definition files carry a ``.pxd`` suffix -* Include files which carry a ``.pxi`` suffix - +This section was moved to :ref:`cython_file_types`. Implementation File =================== @@ -25,805 +23,156 @@ What can it contain? -------------------- -* Basically anything Cythonic, but see below. - What can't it contain? ---------------------- -* There are some restrictions when it comes to **extension types**, if the extension type is - already defined else where... **more on this later** - - Definition File =============== What can it contain? -------------------- -* Any kind of C type declaration. -* ``extern`` C function or variable declarations. -* Declarations for module implementations. -* The definition parts of **extension types**. -* All declarations of functions, etc., for an **external library** - What can't it contain? ---------------------- -* Any non-extern C variable declaration. -* Implementations of C or Python functions. -* Python class definitions -* Python executable statements. -* Any declaration that is defined as **public** to make it accessible to other Cython modules. - - * This is not necessary, as it is automatic. - * a **public** declaration is only needed to make it accessible to **external C code**. - What else? ---------- cimport ``````` -* Use the **cimport** statement, as you would Python's import statement, to access these files - from other definition or implementation files. -* **cimport** does not need to be called in ``.pyx`` file for ``.pxd`` file that has the - same name, as they are already in the same namespace. -* For cimport to find the stated definition file, the path to the file must be appended to the - ``-I`` option of the **Cython compile command**. - compilation order ````````````````` -* When a ``.pyx`` file is to be compiled, Cython first checks to see if a corresponding ``.pxd`` file - exits and processes it first. - - - Include File ============ What can it contain? -------------------- -* Any Cythonic code really, because the entire file is textually embedded at the location - you prescribe. - How do I use it? ---------------- -* Include the ``.pxi`` file with an ``include`` statement like: ``include "spamstuff.pxi`` -* The ``include`` statement can appear anywhere in your Cython file and at any indentation level -* The code in the ``.pxi`` file needs to be rooted at the "zero" indentation level. -* The included code can itself contain other ``include`` statements. - - ==================== Declaring Data Types ==================== - -As a dynamic language, Python encourages a programming style of considering classes and objects in terms of their methods and attributes, more than where they fit into the class hierarchy. - -This can make Python a very relaxed and comfortable language for rapid development, but with a price - the 'red tape' of managing data types is dumped onto the interpreter. At run time, the interpreter does a lot of work searching namespaces, fetching attributes and parsing argument and keyword tuples. This run-time ‘late binding’ is a major cause of Python’s relative slowness compared to ‘early binding’ languages such as C++. - -However with Cython it is possible to gain significant speed-ups through the use of ‘early binding’ programming techniques. - -.. note:: Typing is not a necessity - - Providing static typing to parameters and variables is convenience to speed up your code, but it is not a necessity. Optimize where and when needed. - +This section was moved to :ref:`declaring_data_types`. The cdef Statement ================== -The ``cdef`` statement is used to make C level declarations for: - -:Variables: - -:: - - cdef int i, j, k - cdef float f, g[42], *h - -:Structs: - -:: - - cdef struct Grail: - int age - float volume - -..note Structs can be declared as ``cdef packed struct``, which has -the same effect as the C directive ``#pragma pack(1)``. - -:Unions: - -:: - - cdef union Food: - char *spam - float *eggs - - -:Enums: - -:: - - cdef enum CheeseType: - cheddar, edam, - camembert - -Declaring an enum as ``cpdef`` will create a :pep:`435`-style Python wrapper:: - - cpdef enum CheeseState: - hard = 1 - soft = 2 - runny = 3 - -:Functions: - -:: - - cdef int eggs(unsigned long l, float f): - ... - -:Extension Types: - -:: - - cdef class Spam: - ... - - -.. note:: Constants - - Constants can be defined by using an anonymous enum:: - - cdef enum: - tons_of_spam = 3 - +This section was moved to :ref:`c_variable_and_type_definitions`. Grouping cdef Declarations ========================== -A series of declarations can grouped into a ``cdef`` block:: - - cdef: - struct Spam: - int tons - - int i - float f - Spam *p - - void f(Spam *s): - print s.tons, "Tons of spam" - - -.. note:: ctypedef statement +This section was moved to :ref:`c_variable_and_type_definitions`. - The ``ctypedef`` statement is provided for naming types:: - - ctypedef unsigned long ULong - - ctypedef int *IntPtr +C types and Python classes +========================== +This section was moved to :ref:`types`. Parameters ========== -* Both C and Python **function** types can be declared to have parameters C data types. -* Use normal C declaration syntax:: - - def spam(int i, char *s): - ... - - cdef int eggs(unsigned long l, float f): - ... - -* As these parameters are passed into a Python declared function, they are magically **converted** to the specified C type value. - - * This holds true for only numeric and string types - -* If no type is specified for a parameter or a return value, it is assumed to be a Python object - - * The following takes two Python objects as parameters and returns a Python object:: - - cdef spamobjs(x, y): - ... - - .. note:: -- - - This is different then C language behavior, where it is an int by default. - - - -* Python object types have reference counting performed according to the standard Python C-API rules: - - * Borrowed references are taken as parameters - * New references are returned - -.. todo:: - link or label here the one ref count caveat for NumPy. - -* The name ``object`` can be used to explicitly declare something as a Python Object. - - * For sake of code clarity, it recommended to always use ``object`` explicitly in your code. - - * This is also useful for cases where the name being declared would otherwise be taken for a type:: - - cdef foo(object int): - ... - - * As a return type:: - - cdef object foo(object int): - ... - -.. todo:: - Do a see also here ..?? - -Optional Arguments ------------------- - -* Are supported for ``cdef`` and ``cpdef`` functions -* There are differences though whether you declare them in a ``.pyx`` file or a ``.pxd`` file: - - * When in a ``.pyx`` file, the signature is the same as it is in Python itself:: - - cdef class A: - cdef foo(self): - print "A" - cdef class B(A) - cdef foo(self, x=None) - print "B", x - cdef class C(B): - cpdef foo(self, x=True, int k=3) - print "C", x, k - - - * When in a ``.pxd`` file, the signature is different like this example: ``cdef foo(x=*)``:: - - cdef class A: - cdef foo(self) - cdef class B(A) - cdef foo(self, x=*) - cdef class C(B): - cpdef foo(self, x=*, int k=*) - - - * The number of arguments may increase when subclassing, but the arg types and order must be the same. - -* There may be a slight performance penalty when the optional arg is overridden with one that does not have default values. - -Keyword-only Arguments -======================= - -* As in Python 3, ``def`` functions can have keyword-only arguments listed after a ``"*"`` parameter and before a ``"**"`` parameter if any:: - - def f(a, b, *args, c, d = 42, e, **kwds): - ... - - * Shown above, the ``c``, ``d`` and ``e`` arguments can not be passed as positional arguments and must be passed as keyword arguments. - * Furthermore, ``c`` and ``e`` are required keyword arguments since they do not have a default value. - -* If the parameter name after the ``"*"`` is omitted, the function will not accept any extra positional arguments:: - - def g(a, b, *, c, d): - ... - - * Shown above, the signature takes exactly two positional parameters and has two required keyword parameters - - +This section was moved to :ref:`python_functions_vs_c_functions`. Automatic Type Conversion ========================= -* For basic numeric and string types, in most situations, when a Python object is used in the context of a C value and vice versa. - -* The following table summarizes the conversion possibilities, assuming ``sizeof(int) == sizeof(long)``: - - +----------------------------+--------------------+------------------+ - | C types | From Python types | To Python types | - +============================+====================+==================+ - | [unsigned] char | int, long | int | - +----------------------------+ | | - | [unsigned] short | | | - +----------------------------+ | | - | int, long | | | - +----------------------------+--------------------+------------------+ - | unsigned int | int, long | long | - +----------------------------+ | | - | unsigned long | | | - +----------------------------+ | | - | [unsigned] long long | | | - +----------------------------+--------------------+------------------+ - | float, double, long double | int, long, float | float | - +----------------------------+--------------------+------------------+ - | char * | str/bytes | str/bytes [#]_ | - +----------------------------+--------------------+------------------+ - | struct | | dict | - +----------------------------+--------------------+------------------+ - -.. note:: - **Python String in a C Context** - - * A Python string, passed to C context expecting a ``char*``, is only valid as long as the Python string exists. - * A reference to the Python string must be kept around for as long as the C string is needed. - * If this can't be guaranteed, then make a copy of the C string. - * Cython may produce an error message: ``Obtaining char* from a temporary Python value`` and will not resume compiling in situations like this:: - - cdef char *s - s = pystring1 + pystring2 - - * The reason is that concatenating to strings in Python produces a temporary variable. - - * The variable is decrefed, and the Python string deallocated as soon as the statement has finished, - - * Therefore the lvalue **``s``** is left dangling. - - * The solution is to assign the result of the concatenation to a Python variable, and then obtain the ``char*`` from that:: - - cdef char *s - p = pystring1 + pystring2 - s = p - - .. note:: - **It is up to you to be aware of this, and not to depend on Cython's error message, as it is not guaranteed to be generated for every situation.** - +This section was moved to :ref:`type-conversion`. Type Casting -============= - -* The syntax used in type casting are ``"<"`` and ``">"`` - - .. note:: - The syntax is different from C convention - - :: - - cdef char *p, float *q - p = q - -* If one of the types is a python object for ``x``, Cython will try and do a coercion. - - .. note:: Cython will not stop a casting where there is no conversion, but it will emit a warning. - -* If the address is what is wanted, cast to a ``void*`` first. - - -Type Checking -------------- - -* A cast like ``x`` will cast x to type ``MyExtensionType`` without type checking at all. - -* To have a cast type checked, use the syntax like: ``x``. - - * In this case, Cython will throw an error if ``"x"`` is not a (subclass) of ``MyExtensionType`` +============ -* Automatic type checking for extension types can be obtained whenever ``isinstance()`` is used as the second parameter +This section was moved to :ref:`type_casting`. +Checked Type Casts +------------------ -Python Objects -============== +This section was moved to :ref:`checked_type_casts`. ========================== Statements and Expressions ========================== -* For the most part, control structures and expressions follow Python syntax. -* When applied to Python objects, the semantics are the same unless otherwise noted. -* Most Python operators can be applied to C values with the obvious semantics. -* An expression with mixed Python and C values will have **conversions** performed automatically. -* Python operations are automatically checked for errors, with the appropriate action taken. +This section was moved to :ref:`statements_and_expressions`. Differences Between Cython and C ================================ -* Most notable are C constructs which have no direct equivalent in Python. - - * An integer literal is treated as a C constant - - * It will be truncated to whatever size your C compiler thinks appropriate. - * Cast to a Python object like this:: - - 10000000000000000000 - - * The ``"L"``, ``"LL"`` and the ``"U"`` suffixes have the same meaning as in C - -* There is no ``->`` operator in Cython.. instead of ``p->x``, use ``p.x``. -* There is no ``*`` operator in Cython.. instead of ``*p``, use ``p[0]``. -* ``&`` is permissible and has the same semantics as in C. -* ``NULL`` is the null C pointer. - - * Do NOT use 0. - * ``NULL`` is a reserved word in Cython - -* Syntax for **Type casts** are ``value``. - Scope Rules =========== -* All determination of scoping (local, module, built-in) in Cython is determined statically. -* As with Python, a variable assignment which is not declared explicitly is implicitly declared to be a Python variable residing in the scope where it was assigned. - -.. note:: - * Module-level scope behaves the same way as a Python local scope if you refer to the variable before assigning to it. - - * Tricks, like the following will NOT work in Cython:: - - try: - x = True - except NameError: - True = 1 - - * The above example will not work because ``True`` will always be looked up in the module-level scope. Do the following instead:: - - import __builtin__ - try: - True = __builtin__.True - except AttributeError: - True = 1 - - Built-in Constants ================== -Predefined Python built-in constants: - -* None -* True -* False - - Operator Precedence =================== -* Cython uses Python precedence order, not C - - For-loops ========== -The "for ... in iterable" loop works as in Python, but is even more versatile -in Cython as it can additionally be used on C types. - -* ``range()`` is C optimized when the index value has been declared by ``cdef``, - for example:: - - cdef size_t i - for i in range(n): - ... - -* Iteration over C arrays and sliced pointers is supported and automatically - infers the type of the loop variable, e.g.:: - - cdef double* data = ... - for x in data[:10]: - ... - -* Iterating over many builtin types such as lists and tuples is optimized. - -* There is also a more verbose C-style for-from syntax which, however, is - deprecated in favour of the normal Python "for ... in range()" loop. You - might still find it in legacy code that was written for Pyrex, though. - - * The target expression must be a plain variable name. - - * The name between the lower and upper bounds must be the same as the target name. - - for i from 0 <= i < n: - ... - - * Or when using a step size:: - - for i from 0 <= i < n by s: - ... - - * To reverse the direction, reverse the conditional operation:: - - for i from n > i >= 0: - ... - -* The ``break`` and ``continue`` statements are permissible. - -* Can contain an else clause. - - ===================== Functions and Methods ===================== -* There are three types of function declarations in Cython as the sub-sections show below. -* Only "Python" functions can be called outside a Cython module from *Python interpreted code*. +This section was moved to :ref:`python_functions_vs_c_functions`. Callable from Python (def) ========================== -* Are declared with the ``def`` statement -* Are called with Python objects -* Return Python objects -* See **Parameters** for special consideration - -.. _cdef: - Callable from C (cdef) ====================== -* Are declared with the ``cdef`` statement. -* Are called with either Python objects or C values. -* Can return either Python objects or C values. - -.. _cpdef: - Callable from both Python and C (cpdef) ======================================= -* Are declared with the ``cpdef`` statement. -* Can be called from anywhere, because it uses a little Cython magic. -* Uses the faster C calling conventions when being called from other Cython code. - Overriding ========== -``cpdef`` methods can override ``cdef`` methods:: - - cdef class A: - cdef foo(self): - print "A" - - cdef class B(A) - cdef foo(self, x=None) - print "B", x - - cdef class C(B): - cpdef foo(self, x=True, int k=3) - print "C", x, k - -When subclassing an extension type with a Python class, -``def`` methods can override ``cpdef`` methods but not ``cdef`` -methods:: - - cdef class A: - cdef foo(self): - print("A") - - cdef class B(A): - cpdef foo(self): - print("B") - - class C(B): # NOTE: not cdef class - def foo(self): - print("C") - -If ``C`` above would be an extension type (``cdef class``), -this would not work correctly. -The Cython compiler will give a warning in that case. - +This section was moved to :ref:`overriding_in_extension_types`. Function Pointers ================= -* Functions declared in a ``struct`` are automatically converted to function pointers. -* see **using exceptions with function pointers** - - Python Built-ins ================ -Cython compiles calls to most built-in functions into direct calls to -the corresponding Python/C API routines, making them particularly fast. +This section was moved to :ref:`built_in_functions`. + +Optional Arguments +================== -Only direct function calls using these names are optimised. If you do -something else with one of these names that assumes it's a Python object, -such as assign it to a Python variable, and later call it, the call will -be made as a Python function call. - -+------------------------------+-------------+----------------------------+ -| Function and arguments | Return type | Python/C API Equivalent | -+==============================+=============+============================+ -| abs(obj) | object, | PyNumber_Absolute, fabs, | -| | double, ... | fabsf, ... | -+------------------------------+-------------+----------------------------+ -| callable(obj) | bint | PyObject_Callable | -+------------------------------+-------------+----------------------------+ -| delattr(obj, name) | None | PyObject_DelAttr | -+------------------------------+-------------+----------------------------+ -| exec(code, [glob, [loc]]) | object | - | -+------------------------------+-------------+----------------------------+ -| dir(obj) | list | PyObject_Dir | -+------------------------------+-------------+----------------------------+ -| divmod(a, b) | tuple | PyNumber_Divmod | -+------------------------------+-------------+----------------------------+ -| getattr(obj, name, [default])| object | PyObject_GetAttr | -| (Note 1) | | | -+------------------------------+-------------+----------------------------+ -| hasattr(obj, name) | bint | PyObject_HasAttr | -+------------------------------+-------------+----------------------------+ -| hash(obj) | int / long | PyObject_Hash | -+------------------------------+-------------+----------------------------+ -| intern(obj) | object | Py*_InternFromString | -+------------------------------+-------------+----------------------------+ -| isinstance(obj, type) | bint | PyObject_IsInstance | -+------------------------------+-------------+----------------------------+ -| issubclass(obj, type) | bint | PyObject_IsSubclass | -+------------------------------+-------------+----------------------------+ -| iter(obj, [sentinel]) | object | PyObject_GetIter | -+------------------------------+-------------+----------------------------+ -| len(obj) | Py_ssize_t | PyObject_Length | -+------------------------------+-------------+----------------------------+ -| pow(x, y, [z]) | object | PyNumber_Power | -+------------------------------+-------------+----------------------------+ -| reload(obj) | object | PyImport_ReloadModule | -+------------------------------+-------------+----------------------------+ -| repr(obj) | object | PyObject_Repr | -+------------------------------+-------------+----------------------------+ -| setattr(obj, name) | void | PyObject_SetAttr | -+------------------------------+-------------+----------------------------+ - -Note 1: Pyrex originally provided a function :func:`getattr3(obj, name, default)` -corresponding to the three-argument form of the Python builtin :func:`getattr()`. -Cython still supports this function, but the usage is deprecated in favour of -the normal builtin, which Cython can optimise in both forms. +This section was moved to :ref:`optional_arguments`. +Keyword-only Arguments +======================= + +This section was moved to :ref:`keyword_only_argument`. ============================ Error and Exception Handling ============================ -* A plain ``cdef`` declared function, that does not return a Python object... - - * Has no way of reporting a Python exception to it's caller. - * Will only print a warning message and the exception is ignored. - -* In order to propagate exceptions like this to it's caller, you need to declare an exception value for it. -* There are three forms of declaring an exception for a C compiled program. - - * First:: - - cdef int spam() except -1: - ... - - * In the example above, if an error occurs inside spam, it will immediately return with the value of ``-1``, causing an exception to be propagated to it's caller. - * Functions declared with an exception value, should explicitly prevent a return of that value. - - * Second:: - - cdef int spam() except? -1: - ... - - * Used when a ``-1`` may possibly be returned and is not to be considered an error. - * The ``"?"`` tells Cython that ``-1`` only indicates a *possible* error. - * Now, each time ``-1`` is returned, Cython generates a call to ``PyErr_Occurred`` to verify it is an actual error. - - * Third:: - - cdef int spam() except * - - * A call to ``PyErr_Occurred`` happens *every* time the function gets called. - - .. note:: Returning ``void`` - - A need to propagate errors when returning ``void`` must use this version. - -* Exception values can only be declared for functions returning an.. - - * integer - * enum - * float - * pointer type - * Must be a constant expression - -.. note:: - - .. note:: Function pointers - - * Require the same exception value specification as it's user has declared. - * Use cases here are when used as parameters and when assigned to a variable:: - - int (*grail)(int, char *) except -1 - - .. note:: Python Objects - - * Declared exception values are **not** need. - * Remember that Cython assumes that a function without a declared return value, returns a Python object. - * Exceptions on such functions are implicitly propagated by returning ``NULL`` - - .. note:: C++ - - * For exceptions from C++ compiled programs, see **Wrapping C++ Classes** +This section was moved to :ref:`error_return_values`. Checking return values for non-Cython functions.. ================================================= -* Do not try to raise exceptions by returning the specified value.. Example:: - - cdef extern FILE *fopen(char *filename, char *mode) except NULL # WRONG! - - * The except clause does not work that way. - * It's only purpose is to propagate Python exceptions that have already been raised by either... - - * A Cython function - * A C function that calls Python/C API routines. - -* To propagate an exception for these circumstances you need to raise it yourself:: - - cdef FILE *p - p = fopen("spam.txt", "r") - if p == NULL: - raise SpamError("Couldn't open the spam file") +This section was moved to :ref:`checking_return_values_of_non_cython_functions`. ======================= Conditional Compilation ======================= -* The expressions in the following sub-sections must be valid compile-time expressions. -* They can evaluate to any Python value. -* The *truth* of the result is determined in the usual Python way. +This section was moved to :ref:`conditional_compilation`. Compile-Time Definitions ========================= -* Defined using the ``DEF`` statement:: - - DEF FavouriteFood = "spam" - DEF ArraySize = 42 - DEF OtherArraySize = 2 * ArraySize + 17 - -* The right hand side must be a valid compile-time expression made up of either: - - * Literal values - * Names defined by other ``DEF`` statements - -* They can be combined using any of the Python expression syntax -* Cython provides the following predefined names - - * Corresponding to the values returned by ``os.uname()`` - - * UNAME_SYSNAME - * UNAME_NODENAME - * UNAME_RELEASE - * UNAME_VERSION - * UNAME_MACHINE - -* A name defined by ``DEF`` can appear anywhere an identifier can appear. -* Cython replaces the name with the literal value before compilation. - - * The compile-time expression, in this case, must evaluate to a Python value of ``int``, ``long``, ``float``, or ``str``:: - - cdef int a1[ArraySize] - cdef int a2[OtherArraySize] - print "I like", FavouriteFood - - Conditional Statements ======================= - -* Similar semantics of the C pre-processor -* The following statements can be used to conditionally include or exclude sections of code to compile. - - * ``IF`` - * ``ELIF`` - * ``ELSE`` - -:: - - IF UNAME_SYSNAME == "Windows": - include "icky_definitions.pxi" - ELIF UNAME_SYSNAME == "Darwin": - include "nice_definitions.pxi" - ELIF UNAME_SYSNAME == "Linux": - include "penguin_definitions.pxi" - ELSE: - include "other_definitions.pxi" - -* ``ELIF`` and ``ELSE`` are optional. -* ``IF`` can appear anywhere that a normal statement or declaration can appear -* It can contain any statements or declarations that would be valid in that context. - - * This includes other ``IF`` and ``DEF`` statements - - - -.. [#] The conversion is to/from str for Python 2.x, and bytes for Python 3.x. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/Makefile cython-0.20.1+1~202203241016-9537/docs/src/reference/Makefile --- cython-0.20.1+1~201611251650-6686/docs/src/reference/Makefile 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/Makefile 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d build/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html web htmlhelp latex changes linkcheck - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " web to make files usable by Sphinx.web" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - -clean: - -rm -rf build/* - -html: - mkdir -p build/html build/doctrees - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html - @echo - @echo "Build finished. The HTML pages are in build/html." - -web: - mkdir -p build/web build/doctrees - $(SPHINXBUILD) -b web $(ALLSPHINXOPTS) build/web - @echo - @echo "Build finished; now you can run" - @echo " python -m sphinx.web build/web" - @echo "to start the server." - -htmlhelp: - mkdir -p build/htmlhelp build/doctrees - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) build/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in build/htmlhelp." - -latex: - mkdir -p build/latex build/doctrees - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) build/latex - @echo - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -changes: - mkdir -p build/changes build/doctrees - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) build/changes - @echo - @echo "The overview file is in build/changes." - -linkcheck: - mkdir -p build/linkcheck build/doctrees - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) build/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in build/linkcheck/output.txt." diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/reference/special_methods_table.rst cython-0.20.1+1~202203241016-9537/docs/src/reference/special_methods_table.rst --- cython-0.20.1+1~201611251650-6686/docs/src/reference/special_methods_table.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/reference/special_methods_table.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,220 +1,32 @@ -.. _special_methods_table: - Special Methods Table --------------------- -This table lists all of the special methods together with their parameter and -return types. In the table below, a parameter name of self is used to indicate -that the parameter has the type that the method belongs to. Other parameters -with no type specified in the table are generic Python objects. - -You don't have to declare your method as taking these parameter types. If you -declare different types, conversions will be performed as necessary. +You can find an updated version of the special methods table +in :ref:`special_methods_table`. General ^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __cinit__ |self, ... | | Basic initialisation (no direct Python equivalent) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __init__ |self, ... | | Further initialisation | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __dealloc__ |self | | Basic deallocation (no direct Python equivalent) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __cmp__ |x, y | int | 3-way comparison | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __richcmp__ |x, y, int op | object | Rich comparison (no direct Python equivalent) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __str__ |self | object | str(self) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __repr__ |self | object | repr(self) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __hash__ |self | int | Hash function | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __call__ |self, ... | object | self(...) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __iter__ |self | object | Return iterator for sequence | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getattr__ |self, name | object | Get attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getattribute__ |self, name | object | Get attribute, unconditionally | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __setattr__ |self, name, val | | Set attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __delattr__ |self, name | | Delete attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ +Rich comparison operators +^^^^^^^^^^^^^^^^^^^^^^^^^ Arithmetic operators ^^^^^^^^^^^^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __add__ | x, y | object | binary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __sub__ | x, y | object | binary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mul__ | x, y | object | `*` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __div__ | x, y | object | `/` operator for old-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __floordiv__ | x, y | object | `//` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __truediv__ | x, y | object | `/` operator for new-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mod__ | x, y | object | `%` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __divmod__ | x, y | object | combined div and mod | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pow__ | x, y, z | object | `**` operator or pow(x, y, z) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __neg__ | self | object | unary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pos__ | self | object | unary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __abs__ | self | object | absolute value | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __nonzero__ | self | int | convert to boolean | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __invert__ | self | object | `~` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __lshift__ | x, y | object | `<<` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __rshift__ | x, y | object | `>>` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __and__ | x, y | object | `&` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __or__ | x, y | object | `|` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __xor__ | x, y | object | `^` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - Numeric conversions ^^^^^^^^^^^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __int__ | self | object | Convert to integer | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __long__ | self | object | Convert to long integer | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __float__ | self | object | Convert to float | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __oct__ | self | object | Convert to octal | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __hex__ | self | object | Convert to hexadecimal | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __index__ | self | object | Convert to sequence index | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - In-place arithmetic operators ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __iadd__ | self, x | object | `+=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __isub__ | self, x | object | `-=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __imul__ | self, x | object | `*=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __idiv__ | self, x | object | `/=` operator for old-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ifloordiv__ | self, x | object | `//=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __itruediv__ | self, x | object | `/=` operator for new-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __imod__ | self, x | object | `%=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ipow__ | x, y, z | object | `**=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ilshift__ | self, x | object | `<<=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __irshift__ | self, x | object | `>>=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __iand__ | self, x | object | `&=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ior__ | self, x | object | `|=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ixor__ | self, x | object | `^=` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - Sequences and mappings ^^^^^^^^^^^^^^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __len__ | self int | | len(self) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getitem__ | self, x | object | self[x] | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __setitem__ | self, x, y | | self[x] = y | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __delitem__ | self, x | | del self[x] | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getslice__ | self, Py_ssize_t i, Py_ssize_t j | object | self[i:j] | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __setslice__ | self, Py_ssize_t i, Py_ssize_t j, x | | self[i:j] = x | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __delslice__ | self, Py_ssize_t i, Py_ssize_t j | | del self[i:j] | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __contains__ | self, x | int | x in self | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - Iterators ^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __next__ | self | object | Get next item (called next in Python) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - Buffer interface ^^^^^^^^^^^^^^^^ -.. note:: - The buffer interface is intended for use by C code and is not directly - accessible from Python. It is described in the Python/C API Reference Manual - under sections 6.6 and 10.6. - -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __getreadbuffer__ | self, int i, void `**p` | | | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getwritebuffer__ | self, int i, void `**p` | | | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getsegcount__ | self, int `*p` | | | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __getcharbuffer__ | self, int i, char `**p` | | | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - Descriptor objects ^^^^^^^^^^^^^^^^^^ - -.. note:: - Descriptor objects are part of the support mechanism for new-style - Python classes. See the discussion of descriptors in the Python documentation. - See also :PEP:`252`, "Making Types Look More Like Classes", and :PEP:`253`, - "Subtyping Built-In Types". - -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __get__ | self, instance, class | object | Get value of attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __set__ | self, instance, value | | Set value of attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __delete__ | self, instance | | Delete attribute | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ - - - - - diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/appendix.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/appendix.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/appendix.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/appendix.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,8 +1,8 @@ Appendix: Installing MinGW on Windows ===================================== - 1. Download the MinGW installer from - http://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. + 1. Download the MinGW installer from + https://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. (As of this writing, the download link is a bit difficult to find; it's under "About" in the menu on the left-hand side). You want the file @@ -14,7 +14,7 @@ includes e.g. "c:\\mingw\\bin" (if you installed MinGW to "c:\\mingw"). The following web-page describes the procedure in Windows XP (the Vista procedure is similar): - http://support.microsoft.com/kb/310519 + https://support.microsoft.com/kb/310519 4. Finally, tell Python to use MinGW as the default compiler (otherwise it will try for Visual C). If Python is installed to "c:\\Python27", create a file named @@ -28,4 +28,65 @@ process smoother is welcomed; it is an unfortunate fact that none of the regular Cython developers have convenient access to Windows. -.. [WinInst] http://wiki.cython.org/InstallingOnWindows +Python 3.8+ +----------- + +Since Python 3.8, the search paths of DLL dependencies has been reset. +(`changelog `_) + +Only the system paths, the directory containing the DLL or PYD file +are searched for load-time dependencies. +Instead, a new function `os.add_dll_directory() `_ +was added to supply additional search paths. But such a runtime update is not applicable in all situations. + +Unlike MSVC, MinGW has its owned standard libraries such as ``libstdc++-6.dll``, +which are not placed in the system path (such as ``C:\Windows\System32``). +For a C++ example, you can check the dependencies by MSVC tool ``dumpbin``:: + + > dumpbin /dependents my_gnu_extension.cp38-win_amd64.pyd + ... + Dump of file my_gnu_extension.cp38-win_amd64.pyd + + File Type: DLL + + Image has the following dependencies: + + python38.dll + KERNEL32.dll + msvcrt.dll + libgcc_s_seh-1.dll + libstdc++-6.dll + ... + +These standard libraries can be embedded via static linking, by adding the following options to the linker:: + + -static-libgcc -static-libstdc++ -Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive + +In ``setup.py``, a cross platform config can be added through +extending ``build_ext`` class:: + + from setuptools import setup + from setuptools.command.build_ext import build_ext + + link_args = ['-static-libgcc', + '-static-libstdc++', + '-Wl,-Bstatic,--whole-archive', + '-lwinpthread', + '-Wl,--no-whole-archive'] + + ... # Add extensions + + class Build(build_ext): + def build_extensions(self): + if self.compiler.compiler_type == 'mingw32': + for e in self.extensions: + e.extra_link_args = link_args + super(Build, self).build_extensions() + + setup( + ... + cmdclass={'build_ext': Build}, + ... + ) + +.. [WinInst] https://github.com/cython/cython/wiki/CythonExtensionsOnWindows diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/array.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/array.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/array.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/array.rst 2022-03-24 10:16:46.000000000 +0000 @@ -4,6 +4,9 @@ Working with Python arrays ========================== +.. include:: + ../two-syntax-variants-used + Python has a builtin array module supporting dynamic 1-dimensional arrays of primitive types. It is possible to access the underlying C array of a Python array from within Cython. At the same time they are ordinary Python objects @@ -18,41 +21,39 @@ Safe usage with memory views ---------------------------- -:: - from cpython cimport array - import array - cdef array.array a = array.array('i', [1, 2, 3]) - cdef int[:] ca = a +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/array/safe_usage.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/array/safe_usage.pyx - print ca[0] NB: the import brings the regular Python array object into the namespace while the cimport adds functions accessible from Cython. A Python array is constructed with a type signature and sequence of initial values. For the possible type signatures, refer to the Python -documentation for the `array module `_. +documentation for the `array module `_. Notice that when a Python array is assigned to a variable typed as memory view, there will be a slight overhead to construct the memory view. However, from that point on the variable can be passed to other -functions without overhead, so long as it is typed:: +functions without overhead, so long as it is typed: + + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/array/overhead.py - from cpython cimport array - import array - cdef array.array a = array.array('i', [1, 2, 3]) - cdef int[:] ca = a - - cdef int overhead(object a): - cdef int[:] ca = a - return ca[0] + .. group-tab:: Cython - cdef int no_overhead(int[:] ca): - return ca[0] + .. literalinclude:: ../../examples/tutorial/array/overhead.pyx - print overhead(a) # new memory view will be constructed, overhead - print no_overhead(ca) # ca is already a memory view, so no overhead Zero-overhead, unsafe access to raw C pointer --------------------------------------------- @@ -61,18 +62,16 @@ pointer. There is no type or bounds checking, so be careful to use the right type and signedness. -:: - from cpython cimport array - import array +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/array/unsafe_usage.py - cdef array.array a = array.array('i', [1, 2, 3]) + .. group-tab:: Cython - # access underlying pointer: - print a.data.as_ints[0] + .. literalinclude:: ../../examples/tutorial/array/unsafe_usage.pyx - from libc.string cimport memset - memset(a.data.as_voidptr, 0, len(a) * sizeof(int)) Note that any length-changing operation on the array object may invalidate the pointer. @@ -85,33 +84,30 @@ and preallocate a given number of elements. The array is initialized to zero when requested. -:: - from cpython cimport array - import array +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/array/clone.py - cdef array.array int_array_template = array.array('i', []) - cdef array.array newarray + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/array/clone.pyx - # create an array with 3 elements with same type as template - newarray = array.clone(int_array_template, 3, zero=False) An array can also be extended and resized; this avoids repeated memory reallocation which would occur if elements would be appended or removed one by one. -:: - from cpython cimport array - import array +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/array/resize.py - cdef array.array a = array.array('i', [1, 2, 3]) - cdef array.array b = array.array('i', [4, 5, 6]) + .. group-tab:: Cython - # extend a with b, resize as needed - array.extend(a, b) - # resize a, leaving just original three elements - array.resize(a, len(a) - len(b)) + .. literalinclude:: ../../examples/tutorial/array/resize.pyx API reference @@ -132,6 +128,8 @@ data.as_uints data.as_longs data.as_ulongs + data.as_longlongs # requires Python >=3 + data.as_ulonglongs # requires Python >=3 data.as_floats data.as_doubles data.as_pyunicodes @@ -142,48 +140,142 @@ Functions ~~~~~~~~~ -The following functions are available to Cython from the array module:: +The following functions are available to Cython from the array module - int resize(array self, Py_ssize_t n) except -1 +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1) + def resize(self: array.array, n: cython.Py_ssize_t) -> cython.int + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int resize(array.array self, Py_ssize_t n) except -1 Fast resize / realloc. Not suitable for repeated, small increments; resizes underlying array to exactly the requested amount. -:: +---- + +.. tabs:: + .. group-tab:: Pure Python - int resize_smart(array self, Py_ssize_t n) except -1 + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1) + def resize_smart(self: array.array, n: cython.Py_ssize_t) -> cython.int + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int resize_smart(array.array self, Py_ssize_t n) except -1 Efficient for small increments; uses growth pattern that delivers amortized linear-time appends. -:: +---- + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.inline + def clone(template: array.array, length: cython.Py_ssize_t, zero: cython.bint) -> array.array + + .. group-tab:: Cython + + .. code-block:: cython + + cdef inline array.array clone(array.array template, Py_ssize_t length, bint zero) - cdef inline array clone(array template, Py_ssize_t length, bint zero) Fast creation of a new array, given a template array. Type will be same as ``template``. If zero is ``True``, new array will be initialized with zeroes. -:: +---- + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python - cdef inline array copy(array self) + @cython.cfunc + @cython.inline + def copy(self: array.array) -> array.array + + .. group-tab:: Cython + + .. code-block:: cython + + cdef inline array.array copy(array.array self) Make a copy of an array. -:: +---- + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.inline + @cython.exceptval(-1) + def extend_buffer(self: array.array, stuff: cython.p_char, n: cython.Py_ssize_t) -> cython.int - cdef inline int extend_buffer(array self, char* stuff, Py_ssize_t n) except -1 + .. group-tab:: Cython + + .. code-block:: cython + + cdef inline int extend_buffer(array.array self, char* stuff, Py_ssize_t n) except -1 Efficient appending of new data of same type (e.g. of same array type) ``n``: number of elements (not number of bytes!) -:: +---- + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.inline + @cython.exceptval(-1) + def extend(self: array.array, other: array.array) -> cython.int + + .. group-tab:: Cython - cdef inline int extend(array self, array other) except -1 + .. code-block:: cython + + cdef inline int extend(array.array self, array.array other) except -1 Extend array with data from another array; types must match. -:: +---- + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.inline + def zero(self: array.array) -> cython.void + + .. group-tab:: Cython + + .. code-block:: cython - cdef inline void zero(array self) + cdef inline void zero(array.array self) Set all elements of array to zero. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/caveats.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/caveats.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/caveats.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/caveats.rst 2022-03-24 10:16:46.000000000 +0000 @@ -5,7 +5,6 @@ surprising or unintuitive. Work always goes on to make Cython more natural for Python users, so this list may change in the future. - - ``10**-2 == 0``, instead of ``0.01`` like in Python. - Given two typed ``int`` variables ``a`` and ``b``, ``a % b`` has the same sign as the second argument (following Python semantics) rather than having the same sign as the first (as in C). The C behavior can be @@ -15,5 +14,5 @@ print(range(-n, n))`` will print an empty list, since ``-n`` wraps around to a large positive integer prior to being passed to the ``range`` function. - - Python's ``float`` type actually wraps C ``double`` values, and + - Python's ``float`` type actually wraps C ``double`` values, and the ``int`` type in Python 2.x wraps C ``long`` values. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/cdef_classes.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/cdef_classes.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/cdef_classes.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/cdef_classes.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,21 +1,19 @@ +*********************************** Extension types (aka. cdef classes) -=================================== +*********************************** -To support object-oriented programming, Cython supports writing normal -Python classes exactly as in Python:: +.. include:: + ../two-syntax-variants-used - class MathFunction(object): - def __init__(self, name, operator): - self.name = name - self.operator = operator +To support object-oriented programming, Cython supports writing normal +Python classes exactly as in Python: - def __call__(self, *operands): - return self.operator(*operands) +.. literalinclude:: ../../examples/tutorial/cdef_classes/math_function.py Based on what Python calls a "built-in type", however, Cython supports a second kind of class: *extension types*, sometimes referred to as -"cdef classes" due to the keywords used for their declaration. They -are somewhat restricted compared to Python classes, but are generally +"cdef classes" due to the Cython language keywords used for their declaration. +They are somewhat restricted compared to Python classes, but are generally more memory efficient and faster than generic Python classes. The main difference is that they use a C struct to store their fields and methods instead of a Python dict. This allows them to store arbitrary C types @@ -30,45 +28,68 @@ inherit from any number of Python classes and extension types, both in Cython code and pure Python code. -So far our integration example has not been very useful as it only -integrates a single hard-coded function. In order to remedy this, -with hardly sacrificing speed, we will use a cdef class to represent a -function on floating point numbers:: - - cdef class Function: - cpdef double evaluate(self, double x) except *: - return 0 - -The directive cpdef makes two versions of the method available; one -fast for use from Cython and one slower for use from Python. Then:: - - cdef class SinOfSquareFunction(Function): - cpdef double evaluate(self, double x) except *: - return sin(x**2) +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/math_function_2.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/math_function_2.pyx + +The ``cpdef`` command (or ``@cython.ccall`` in Python syntax) makes two versions +of the method available; one fast for use from Cython and one slower for use +from Python. + +Now we can add subclasses of the ``Function`` class that implement different +math functions in the same ``evaluate()`` method. + +Then: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.py + :caption: sin_of_square.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.pyx + :caption: sin_of_square.pyx This does slightly more than providing a python wrapper for a cdef -method: unlike a cdef method, a cpdef method is fully overrideable by -methods and instance attributes in Python subclasses. It adds a +method: unlike a cdef method, a cpdef method is fully overridable by +methods and instance attributes in Python subclasses. This adds a little calling overhead compared to a cdef method. -Using this, we can now change our integration example:: +To make the class definitions visible to other modules, and thus allow for +efficient C-level usage and inheritance outside of the module that +implements them, we define them in a ``.pxd`` file with the same name +as the module. Note that we are using Cython syntax here, not Python syntax. - def integrate(Function f, double a, double b, int N): - cdef int i - cdef double s, dx - if f is None: - raise ValueError("f cannot be None") - s = 0 - dx = (b-a)/N - for i in range(N): - s += f.evaluate(a+i*dx) - return s * dx - - print(integrate(SinOfSquareFunction(), 0, 1, 10000)) - -This is almost as fast as the previous code, however it is much more flexible -as the function to integrate can be changed. We can even pass in a new -function defined in Python-space:: +.. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.pxd + :caption: sin_of_square.pxd + +With this way to implement different functions as subclasses with fast, +Cython callable methods, we can now pass these ``Function`` objects into +an algorithm for numeric integration, that evaluates an arbitrary user +provided function over a value interval. + +Using this, we can now change our integration example: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/integrate.py + :caption: integrate.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/integrate.pyx + :caption: integrate.pyx + +We can even pass in a new ``Function`` defined in Python space, which overrides +the Cython implemented method of the base class:: >>> import integrate >>> class MyPolynomial(integrate.Function): @@ -78,70 +99,58 @@ >>> integrate(MyPolynomial(), 0, 1, 10000) -7.8335833300000077 -This is about 20 times slower, but still about 10 times faster than -the original Python-only integration code. This shows how large the -speed-ups can easily be when whole loops are moved from Python code -into a Cython module. +Since ``evaluate()`` is a Python method here, which requires Python objects +as input and output, this is several times slower than the straight C call +to the Cython method, but still faster than a plain Python variant. +This shows how large the speed-ups can easily be when whole computational +loops are moved from Python code into a Cython module. Some notes on our new implementation of ``evaluate``: - - The fast method dispatch here only works because ``evaluate`` was - declared in ``Function``. Had ``evaluate`` been introduced in - ``SinOfSquareFunction``, the code would still work, but Cython - would have used the slower Python method dispatch mechanism - instead. - - - In the same way, had the argument ``f`` not been typed, but only - been passed as a Python object, the slower Python dispatch would - be used. - - - Since the argument is typed, we need to check whether it is - ``None``. In Python, this would have resulted in an ``AttributeError`` - when the ``evaluate`` method was looked up, but Cython would instead - try to access the (incompatible) internal structure of ``None`` as if - it were a ``Function``, leading to a crash or data corruption. +- The fast method dispatch here only works because ``evaluate`` was + declared in ``Function``. Had ``evaluate`` been introduced in + ``SinOfSquareFunction``, the code would still work, but Cython + would have used the slower Python method dispatch mechanism + instead. + +- In the same way, had the argument ``f`` not been typed, but only + been passed as a Python object, the slower Python dispatch would + be used. + +- Since the argument is typed, we need to check whether it is + ``None``. In Python, this would have resulted in an ``AttributeError`` + when the ``evaluate`` method was looked up, but Cython would instead + try to access the (incompatible) internal structure of ``None`` as if + it were a ``Function``, leading to a crash or data corruption. There is a *compiler directive* ``nonecheck`` which turns on checks for this, at the cost of decreased speed. Here's how compiler directives -are used to dynamically switch on or off ``nonecheck``:: - - #cython: nonecheck=True - # ^^^ Turns on nonecheck globally +are used to dynamically switch on or off ``nonecheck``: - import cython +.. tabs:: + .. group-tab:: Pure Python - # Turn off nonecheck locally for the function - @cython.nonecheck(False) - def func(): - cdef MyClass obj = None - try: - # Turn nonecheck on again for a block - with cython.nonecheck(True): - print obj.myfunc() # Raises exception - except AttributeError: - pass - print obj.myfunc() # Hope for a crash! + .. literalinclude:: ../../examples/tutorial/cdef_classes/nonecheck.py + :caption: nonecheck.py + .. group-tab:: Cython + .. literalinclude:: ../../examples/tutorial/cdef_classes/nonecheck.pyx + :caption: nonecheck.pyx Attributes in cdef classes behave differently from attributes in regular classes: - - All attributes must be pre-declared at compile-time - - Attributes are by default only accessible from Cython (typed access) - - Properties can be declared to expose dynamic attributes to Python-space - -:: - - cdef class WaveFunction(Function): - # Not available in Python-space: - cdef double offset - # Available in Python-space: - cdef public double freq - # Available in Python-space: - @property - def period(self): - return 1.0 / self.freq - @period.setter - def period(self, value): - self.freq = 1.0 / value - <...> +- All attributes must be pre-declared at compile-time +- Attributes are by default only accessible from Cython (typed access) +- Properties can be declared to expose dynamic attributes to Python-space + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/wave_function.py + :caption: wave_function.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/wave_function.pyx + :caption: wave_function.pyx diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/clibraries.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/clibraries.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/clibraries.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/clibraries.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,12 @@ + +.. _using_c_libraries: + +****************** Using C libraries -================= +****************** + +.. include:: + ../two-syntax-variants-used Apart from writing fast code, one of the main use cases of Cython is to call external C libraries from Python code. As Cython code @@ -20,55 +27,26 @@ handling easier, however, you decide to wrap it in a Python extension type that can encapsulate all memory management. -.. [CAlg] Simon Howard, C Algorithms library, http://c-algorithms.sourceforge.net/ +.. [CAlg] Simon Howard, C Algorithms library, https://fragglet.github.io/c-algorithms/ Defining external declarations ------------------------------- - -The C API of the queue implementation, which is defined in the header -file ``libcalg/queue.h``, essentially looks like this:: - - /* file: queue.h */ - - typedef struct _Queue Queue; - typedef void *QueueValue; +============================== - Queue *queue_new(void); - void queue_free(Queue *queue); +You can download CAlg `here `_. - int queue_push_head(Queue *queue, QueueValue data); - QueueValue queue_pop_head(Queue *queue); - QueueValue queue_peek_head(Queue *queue); - - int queue_push_tail(Queue *queue, QueueValue data); - QueueValue queue_pop_tail(Queue *queue); - QueueValue queue_peek_tail(Queue *queue); +The C API of the queue implementation, which is defined in the header +file :file:`c-algorithms/src/queue.h`, essentially looks like this: - int queue_is_empty(Queue *queue); +.. literalinclude:: ../../examples/tutorial/clibraries/c-algorithms/src/queue.h + :language: C + :caption: queue.h To get started, the first step is to redefine the C API in a ``.pxd`` -file, say, ``cqueue.pxd``:: +file, say, :file:`cqueue.pxd`: - # file: cqueue.pxd - - cdef extern from "libcalg/queue.h": - ctypedef struct Queue: - pass - ctypedef void* QueueValue - - Queue* queue_new() - void queue_free(Queue* queue) - - int queue_push_head(Queue* queue, QueueValue data) - QueueValue queue_pop_head(Queue* queue) - QueueValue queue_peek_head(Queue* queue) - - int queue_push_tail(Queue* queue, QueueValue data) - QueueValue queue_pop_tail(Queue* queue) - QueueValue queue_peek_tail(Queue* queue) - - bint queue_is_empty(Queue* queue) +.. literalinclude:: ../../examples/tutorial/clibraries/cqueue.pxd + :caption: cqueue.pxd Note how these declarations are almost identical to the header file declarations, so you can often just copy them over. However, you do @@ -123,31 +101,37 @@ Writing a wrapper class ------------------------ +======================= After declaring our C library's API, we can start to design the Queue class that should wrap the C queue. It will live in a file called -``queue.pyx``. [#]_ +:file:`queue.pyx`/:file:`queue.py`. [#]_ -.. [#] Note that the name of the ``.pyx`` file must be different from - the ``cqueue.pxd`` file with declarations from the C library, +.. [#] Note that the name of the ``.pyx``/``.py`` file must be different from + the :file:`cqueue.pxd` file with declarations from the C library, as both do not describe the same code. A ``.pxd`` file next to - a ``.pyx`` file with the same name defines exported - declarations for code in the ``.pyx`` file. As the - ``cqueue.pxd`` file contains declarations of a regular C - library, there must not be a ``.pyx`` file with the same name + a ``.pyx``/``.py`` file with the same name defines exported + declarations for code in the ``.pyx``/``.py`` file. As the + :file:`cqueue.pxd` file contains declarations of a regular C + library, there must not be a ``.pyx``/``.py`` file with the same name that Cython associates with it. -Here is a first start for the Queue class:: +Here is a first start for the Queue class: - # file: queue.pyx +.. tabs:: - cimport cqueue + .. group-tab:: Pure Python - cdef class Queue: - cdef cqueue.Queue* _c_queue - def __cinit__(self): - self._c_queue = cqueue.queue_new() + .. literalinclude:: ../../examples/tutorial/clibraries/queue.py + :caption: queue.py + + .. note:: Currently, Cython contains a bug not allowing using + annotations with types containing pointers (GitHub issue :issue:`4293`). + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/clibraries/queue.pyx + :caption: queue.pyx Note that it says ``__cinit__`` rather than ``__init__``. While ``__init__`` is available as well, it is not guaranteed to be run (for @@ -156,10 +140,11 @@ leads to hard crashes of the Python interpreter, Cython provides ``__cinit__`` which is *always* called immediately on construction, before CPython even considers calling ``__init__``, and which -therefore is the right place to initialise ``cdef`` fields of the new -instance. However, as ``__cinit__`` is called during object -construction, ``self`` is not fully constructed yet, and one must -avoid doing anything with ``self`` but assigning to ``cdef`` fields. +therefore is the right place to initialise static attributes +(``cdef`` fields) of the new instance. However, as ``__cinit__`` is +called during object construction, ``self`` is not fully constructed yet, +and one must avoid doing anything with ``self`` but assigning to static +attributes (``cdef`` fields). Note also that the above method takes no parameters, although subtypes may want to accept some. A no-arguments ``__cinit__()`` method is a @@ -172,7 +157,7 @@ Memory management ------------------ +================= Before we continue implementing the other methods, it is important to understand that the above implementation is not safe. In case @@ -184,22 +169,25 @@ pointer to the new queue. The Python way to get out of this is to raise a ``MemoryError`` [#]_. -We can thus change the init function as follows:: +We can thus change the init function as follows: - cimport cqueue +.. tabs:: - cdef class Queue: - cdef cqueue.Queue* _c_queue - def __cinit__(self): - self._c_queue = cqueue.queue_new() - if self._c_queue is NULL: - raise MemoryError() + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/clibraries/queue2.py + :caption: queue.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/clibraries/queue2.pyx + :caption: queue.pyx .. [#] In the specific case of a ``MemoryError``, creating a new exception instance in order to raise it may actually fail because we are running out of memory. Luckily, CPython provides a C-API function ``PyErr_NoMemory()`` that safely raises the right - exception for us. Since version 0.14.1, Cython automatically + exception for us. Cython automatically substitutes this C-API call whenever you write ``raise MemoryError`` or ``raise MemoryError()``. If you use an older version, you have to cimport the C-API function from the standard @@ -210,59 +198,236 @@ end, CPython provides a callback that Cython makes available as a special method ``__dealloc__()``. In our case, all we have to do is to free the C Queue, but only if we succeeded in initialising it in -the init method:: +the init method: + +.. tabs:: + + .. group-tab:: Pure Python - def __dealloc__(self): - if self._c_queue is not NULL: - cqueue.queue_free(self._c_queue) + .. code-block:: python + def __dealloc__(self): + if self._c_queue is not cython.NULL: + cqueue.queue_free(self._c_queue) + + .. group-tab:: Cython + + .. code-block:: cython + + def __dealloc__(self): + if self._c_queue is not NULL: + cqueue.queue_free(self._c_queue) Compiling and linking ---------------------- +===================== At this point, we have a working Cython module that we can test. To -compile it, we need to configure a ``setup.py`` script for distutils. -Here is the most basic script for compiling a Cython module:: +compile it, we need to configure a ``setup.py`` script for setuptools. +Here is the most basic script for compiling a Cython module + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + from setuptools import Extension, setup + from Cython.Build import cythonize + + setup( + ext_modules = cythonize([Extension("queue", ["queue.py"])]) + ) + + .. group-tab:: Cython + + .. code-block:: cython + + from setuptools import Extension, setup + from Cython.Build import cythonize + + setup( + ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) + ) + + +To build against the external C library, we need to make sure Cython finds the necessary libraries. +There are two ways to archive this. First we can tell setuptools where to find +the c-source to compile the :file:`queue.c` implementation automatically. Alternatively, +we can build and install C-Alg as system library and dynamically link it. The latter is useful +if other applications also use C-Alg. + + +Static Linking +--------------- + +To build the c-code automatically we need to include compiler directives in :file:`queue.pyx`/:file:`queue.py` + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + # distutils: sources = c-algorithms/src/queue.c + # distutils: include_dirs = c-algorithms/src/ + + import cython + from cython.cimports import cqueue + + @cython.cclass + class Queue: + _c_queue = cython.declare(cython.pointer(cqueue.Queue)) + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is cython.NULL: + raise MemoryError() + + def __dealloc__(self): + if self._c_queue is not cython.NULL: + cqueue.queue_free(self._c_queue) + + .. group-tab:: Cython + + .. code-block:: cython + + # distutils: sources = c-algorithms/src/queue.c + # distutils: include_dirs = c-algorithms/src/ + + + cimport cqueue + + + cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is NULL: + raise MemoryError() + + def __dealloc__(self): + if self._c_queue is not NULL: + cqueue.queue_free(self._c_queue) + +The ``sources`` compiler directive gives the path of the C +files that setuptools is going to compile and +link (statically) into the resulting extension module. +In general all relevant header files should be found in ``include_dirs``. +Now we can build the project using: + +.. code-block:: bash + + $ python setup.py build_ext -i + +And test whether our build was successful: + +.. code-block:: bash + + $ python -c 'import queue; Q = queue.Queue()' + + +Dynamic Linking +--------------- + +Dynamic linking is useful, if the library we are going to wrap is already +installed on the system. To perform dynamic linking we first need to +build and install c-alg. + +To build c-algorithms on your system: + +.. code-block:: bash + + $ cd c-algorithms + $ sh autogen.sh + $ ./configure + $ make + +to install CAlg run: + +.. code-block:: bash + + $ make install - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize +Afterwards the file :file:`/usr/local/lib/libcalg.so` should exist. - setup( - ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) - ) +.. note:: -To build against the external C library, we must extend this script to -include the necessary setup. Assuming the library is installed in the -usual places (e.g. under ``/usr/lib`` and ``/usr/include`` on a -Unix-like system), we could simply change the extension setup from + This path applies to Linux systems and may be different on other platforms, + so you will need to adapt the rest of the tutorial depending on the path + where ``libcalg.so`` or ``libcalg.dll`` is on your system. -:: +In this approach we need to tell the setup script to link with an external library. +To do so we need to extend the setup script to install change the extension setup from - ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + ext_modules = cythonize([Extension("queue", ["queue.py"])]) + + .. group-tab:: Cython + + .. code-block:: cython + + ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) to -:: +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + ext_modules = cythonize([ + Extension("queue", ["queue.py"], + libraries=["calg"]) + ]) + + .. group-tab:: Cython - ext_modules = cythonize([ - Extension("queue", ["queue.pyx"], - libraries=["calg"]) - ]) + .. code-block:: cython -If it is not installed in a 'normal' location, users can provide the + ext_modules = cythonize([ + Extension("queue", ["queue.pyx"], + libraries=["calg"]) + ]) + +Now we should be able to build the project using: + +.. code-block:: bash + + $ python setup.py build_ext -i + +If the `libcalg` is not installed in a 'normal' location, users can provide the required parameters externally by passing appropriate C compiler -flags, such as:: +flags, such as: + +.. code-block:: bash CFLAGS="-I/usr/local/otherdir/calg/include" \ LDFLAGS="-L/usr/local/otherdir/calg/lib" \ python setup.py build_ext -i + + +Before we run the module, we also need to make sure that `libcalg` is in +the `LD_LIBRARY_PATH` environment variable, e.g. by setting: + +.. code-block:: bash + + $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + Once we have compiled the module for the first time, we can now import -it and instantiate a new Queue:: +it and instantiate a new Queue: + +.. code-block:: bash $ export PYTHONPATH=. - $ python -c 'import queue.Queue as Q ; Q()' + $ python -c 'import queue; Q = queue.Queue()' However, this is all our Queue class can do so far, so let's make it more usable. @@ -277,7 +442,7 @@ queue, it's enough to provide the methods ``append()``, ``peek()`` and ``pop()``, and additionally an ``extend()`` method to add multiple values at once. Also, since we already know that all values will be -coming from C, it's best to provide only ``cdef`` methods for now, and +coming from C, it's best to provide only ``cdef``/``@cfunc`` methods for now, and to give them a straight C interface. In C, it is common for data structures to store data as a ``void*`` to @@ -287,43 +452,116 @@ to ``void*`` and vice versa, and store the value directly as the pointer value. -Here is a simple implementation for the ``append()`` method:: +Here is a simple implementation for the ``append()`` method: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python - cdef append(self, int value): - cqueue.queue_push_tail(self._c_queue, value) + @cython.cfunc + def append(self, value: cython.int): + cqueue.queue_push_tail(self._c_queue, cython.cast(cython.p_void, value)) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef append(self, int value): + cqueue.queue_push_tail(self._c_queue, value) Again, the same error handling considerations as for the ``__cinit__()`` method apply, so that we end up with this -implementation instead:: +implementation instead: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def append(self, value: cython.int): + if not cqueue.queue_push_tail(self._c_queue, + cython.cast(cython.p_void, value)): + raise MemoryError() + + .. group-tab:: Cython + + .. code-block:: cython + + cdef append(self, int value): + if not cqueue.queue_push_tail(self._c_queue, + value): + raise MemoryError() + +Adding an ``extend()`` method should now be straight forward: + +.. tabs:: - cdef append(self, int value): - if not cqueue.queue_push_tail(self._c_queue, - value): - raise MemoryError() - -Adding an ``extend()`` method should now be straight forward:: - - cdef extend(self, int* values, size_t count): - """Append all ints to the queue. - """ - cdef size_t i - for i in range(count): - if not cqueue.queue_push_tail( - self._c_queue, values[i]): - raise MemoryError() + .. group-tab:: Pure Python -This becomes handy when reading values from a NumPy array, for -example. + .. code-block:: python + + @cython.cfunc + def extend(self, values: cython.p_int, count: cython.size_t): + """Append all ints to the queue. + """ + value: cython.int + for value in values[:count]: # Slicing pointer to limit the iteration boundaries. + self.append(value) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef extend(self, int* values, size_t count): + """Append all ints to the queue. + """ + cdef int value + for value in values[:count]: # Slicing pointer to limit the iteration boundaries. + self.append(value) + +This becomes handy when reading values from a C array, for example. So far, we can only add data to the queue. The next step is to write the two methods to get the first element: ``peek()`` and ``pop()``, -which provide read-only and destructive read access respectively:: +which provide read-only and destructive read access respectively. +To avoid compiler warnings when casting ``void*`` to ``int`` directly, +we use an intermediate data type that is big enough to hold a ``void*``. +Here, ``Py_ssize_t``: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def peek(self) -> cython.int: + return cython.cast(cython.Py_ssize_t, cqueue.queue_peek_head(self._c_queue)) + + @cython.cfunc + def pop(self) -> cython.int: + return cython.cast(cython.Py_ssize_t, cqueue.queue_pop_head(self._c_queue)) - cdef int peek(self): - return cqueue.queue_peek_head(self._c_queue) + .. group-tab:: Cython - cdef int pop(self): - return cqueue.queue_pop_head(self._c_queue) + .. code-block:: cython + + cdef int peek(self): + return cqueue.queue_peek_head(self._c_queue) + + cdef int pop(self): + return cqueue.queue_pop_head(self._c_queue) + +Normally, in C, we risk losing data when we convert a larger integer type +to a smaller integer type without checking the boundaries, and ``Py_ssize_t`` +may be a larger type than ``int``. But since we control how values are added +to the queue, we already know that all values that are in the queue fit into +an ``int``, so the above conversion from ``void*`` to ``Py_ssize_t`` to ``int`` +(the return type) is safe by design. Handling errors @@ -331,28 +569,50 @@ Now, what happens when the queue is empty? According to the documentation, the functions return a ``NULL`` pointer, which is -typically not a valid value. Since we are simply casting to and +typically not a valid value. But since we are simply casting to and from ints, we cannot distinguish anymore if the return value was ``NULL`` because the queue was empty or because the value stored in -the queue was ``0``. However, in Cython code, we would expect the -first case to raise an exception, whereas the second case should -simply return ``0``. To deal with this, we need to special case this -value, and check if the queue really is empty or not:: - - cdef int peek(self) except? -1: - value = cqueue.queue_peek_head(self._c_queue) - if value == 0: - # this may mean that the queue is empty, or - # that it happens to contain a 0 value - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return value +the queue was ``0``. In Cython code, we want the first case to +raise an exception, whereas the second case should simply return +``0``. To deal with this, we need to special case this value, +and check if the queue really is empty or not: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1, check=True) + def peek(self) -> cython.int: + value: cython.int = cython.cast(cython.Py_ssize_t, cqueue.queue_peek_head(self._c_queue)) + if value == 0: + # this may mean that the queue is empty, or + # that it happens to contain a 0 value + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return value + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int peek(self) except? -1: + cdef int value = cqueue.queue_peek_head(self._c_queue) + if value == 0: + # this may mean that the queue is empty, or + # that it happens to contain a 0 value + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return value Note how we have effectively created a fast path through the method in the hopefully common cases that the return value is not ``0``. Only that specific case needs an additional check if the queue is empty. -The ``except? -1`` declaration in the method signature falls into the +The ``except? -1`` or ``@cython.exceptval(-1, check=True)`` declaration +in the method signature falls into the same category. If the function was a Python function returning a Python object value, CPython would simply return ``NULL`` internally instead of a Python object to indicate an exception, which would @@ -374,7 +634,8 @@ We chose to use ``-1`` as the exception return value as we expect it to be an unlikely value to be put into the queue. The question mark -in the ``except? -1`` declaration indicates that the return value is +in the ``except? -1`` declaration and ``check=True`` in ``@cython.exceptval`` +indicates that the return value is ambiguous (there *may* be a ``-1`` value in the queue, after all) and that an additional exception check using ``PyErr_Occurred()`` is needed in calling code. Without it, Cython code that calls this @@ -387,12 +648,29 @@ Now that the ``peek()`` method is implemented, the ``pop()`` method also needs adaptation. Since it removes a value from the queue, however, it is not enough to test if the queue is empty *after* the -removal. Instead, we must test it on entry:: +removal. Instead, we must test it on entry: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1, check=True) + def pop(self) -> cython.int: + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return cython.cast(cython.Py_ssize_t, cqueue.queue_pop_head(self._c_queue)) - cdef int pop(self) except? -1: - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return cqueue.queue_pop_head(self._c_queue) + .. group-tab:: Cython + + .. code-block:: cython + + cdef int pop(self) except? -1: + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return cqueue.queue_pop_head(self._c_queue) The return value for exception propagation is declared exactly as for ``peek()``. @@ -407,7 +685,7 @@ Note that this method returns either ``True`` or ``False`` as we declared the return type of the ``queue_is_empty()`` function as -``bint`` in ``cqueue.pxd``. +``bint`` in :file:`cqueue.pxd`. Testing the result @@ -421,86 +699,47 @@ not callable from doctests. A quick way to provide a Python API for the class is to change the -methods from ``cdef`` to ``cpdef``. This will let Cython generate two -entry points, one that is callable from normal Python code using the -Python call semantics and Python objects as arguments, and one that is -callable from C code with fast C semantics and without requiring -intermediate argument conversion from or to Python types. Note that ``cpdef`` -methods ensure that they can be appropriately overridden by Python -methods even when they are called from Cython. This adds a tiny overhead -compared to ``cdef`` methods. +methods from ``cdef``/``@cfunc`` to ``cpdef``/``@ccall``. This will +let Cython generate two entry points, one that is callable from normal +Python code using the Python call semantics and Python objects as arguments, +and one that is callable from C code with fast C semantics and without requiring +intermediate argument conversion from or to Python types. Note that +``cpdef``/``@ccall`` methods ensure that they can be appropriately overridden +by Python methods even when they are called from Cython. This adds a tiny overhead +compared to ``cdef``/``@cfunc`` methods. + +Now that we have both a C-interface and a Python interface for our +class, we should make sure that both interfaces are consistent. +Python users would expect an ``extend()`` method that accepts arbitrary +iterables, whereas C users would like to have one that allows passing +C arrays and C memory. Both signatures are incompatible. + +We will solve this issue by considering that in C, the API could also +want to support other input types, e.g. arrays of ``long`` or ``char``, +which is usually supported with differently named C API functions such as +``extend_ints()``, ``extend_longs()``, ``extend_chars()``, etc. This allows +us to free the method name ``extend()`` for the duck typed Python method, +which can accept arbitrary iterables. The following listing shows the complete implementation that uses -``cpdef`` methods where possible:: +``cpdef``/``@ccall`` methods where possible: - cimport cqueue +.. tabs:: - cdef class Queue: - """A queue class for C integer values. + .. group-tab:: Pure Python - >>> q = Queue() - >>> q.append(5) - >>> q.peek() - 5 - >>> q.pop() - 5 - """ - cdef cqueue.Queue* _c_queue - def __cinit__(self): - self._c_queue = cqueue.queue_new() - if self._c_queue is NULL: - raise MemoryError() - - def __dealloc__(self): - if self._c_queue is not NULL: - cqueue.queue_free(self._c_queue) - - cpdef append(self, int value): - if not cqueue.queue_push_tail(self._c_queue, - value): - raise MemoryError() - - cdef extend(self, int* values, size_t count): - cdef size_t i - for i in xrange(count): - if not cqueue.queue_push_tail( - self._c_queue, values[i]): - raise MemoryError() + .. literalinclude:: ../../examples/tutorial/clibraries/queue3.py + :caption: queue.py - cpdef int peek(self) except? -1: - cdef int value = \ - cqueue.queue_peek_head(self._c_queue) - if value == 0: - # this may mean that the queue is empty, - # or that it happens to contain a 0 value - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return value + .. group-tab:: Cython - cpdef int pop(self) except? -1: - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return cqueue.queue_pop_head(self._c_queue) - - def __bool__(self): - return not cqueue.queue_is_empty(self._c_queue) - -The ``cpdef`` feature is obviously not available for the ``extend()`` -method, as the method signature is incompatible with Python argument -types. However, if wanted, we can rename the C-ish ``extend()`` -method to e.g. ``c_extend()``, and write a new ``extend()`` method -instead that accepts an arbitrary Python iterable:: - - cdef c_extend(self, int* values, size_t count): - cdef size_t i - for i in range(count): - if not cqueue.queue_push_tail( - self._c_queue, values[i]): - raise MemoryError() + .. literalinclude:: ../../examples/tutorial/clibraries/queue3.pyx + :caption: queue.pyx + +Now we can test our Queue implementation using a python script, +for example here :file:`test_queue.py`: - cpdef extend(self, values): - for value in values: - self.append(value) +.. literalinclude:: ../../examples/tutorial/clibraries/test_queue.py As a quick test with 10000 numbers on the author's machine indicates, using this Queue from Cython code with C ``int`` values is about five @@ -546,29 +785,73 @@ predicate function. First, we have to define a callback function with the expected -signature that we can pass into the C-API function:: +signature that we can pass into the C-API function: - cdef int evaluate_predicate(void* context, cqueue.QueueValue value): - "Callback function that can be passed as predicate_func" - try: - # recover Python function object from void* argument - func = context - # call function, convert result into 0/1 for True/False - return bool(func(value)) - except: - # catch any Python errors and return error indicator - return -1 +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(check=False) + def evaluate_predicate(context: cython.p_void, value: cqueue.QueueValue) -> cython.int: + "Callback function that can be passed as predicate_func" + try: + # recover Python function object from void* argument + func = cython.cast(object, context) + # call function, convert result into 0/1 for True/False + return bool(func(cython.cast(int, value))) + except: + # catch any Python errors and return error indicator + return -1 + + .. note:: ``@cfunc`` functions in pure python are defined as ``@exceptval(-1, check=True)`` + by default. Since ``evaluate_predicate()`` should be passed to function as parameter, + we need to turn off exception checking entirely. + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int evaluate_predicate(void* context, cqueue.QueueValue value): + "Callback function that can be passed as predicate_func" + try: + # recover Python function object from void* argument + func = context + # call function, convert result into 0/1 for True/False + return bool(func(value)) + except: + # catch any Python errors and return error indicator + return -1 The main idea is to pass a pointer (a.k.a. borrowed reference) to the function object as the user context argument. We will call the C-API -function as follows:: +function as follows: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def pop_until(self, python_predicate_function): + result = cqueue.queue_pop_head_until( + self._c_queue, evaluate_predicate, + cython.cast(cython.p_void, python_predicate_function)) + if result == -1: + raise RuntimeError("an error occurred") + + .. group-tab:: Cython + + .. code-block:: cython - def pop_until(self, python_predicate_function): - result = cqueue.queue_pop_head_until( - self._c_queue, evaluate_predicate, - python_predicate_function) - if result == -1: - raise RuntimeError("an error occurred") + def pop_until(self, python_predicate_function): + result = cqueue.queue_pop_head_until( + self._c_queue, evaluate_predicate, + python_predicate_function) + if result == -1: + raise RuntimeError("an error occurred") The usual pattern is to first cast the Python object reference into a :c:type:`void*` to pass it into the C-API function, and then cast diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/cython_tutorial.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/cython_tutorial.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/cython_tutorial.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/cython_tutorial.rst 2022-03-24 10:16:46.000000000 +0000 @@ -6,6 +6,9 @@ Basic Tutorial ************** +.. include:: + ../two-syntax-variants-used + The Basics of Cython ==================== @@ -18,7 +21,7 @@ equivalent calls to the Python/C API. But Cython is much more than that, because parameters and variables can be -declared to have C data types. Code which manipulates Python values and C +declared to have C data types. Code which manipulates :term:`Python values` and C values can be freely intermixed, with conversions occurring automatically wherever possible. Reference count maintenance and error checking of Python operations is also automatic, and the full power of Python's exception @@ -34,13 +37,13 @@ So lets start with the canonical python hello world:: - print "Hello World" + print("Hello World") Save this code in a file named :file:`helloworld.pyx`. Now we need to create the :file:`setup.py`, which is like a python Makefile (for more information see :ref:`compilation`). Your :file:`setup.py` should look like:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -49,7 +52,7 @@ To use this to build your Cython file use the commandline options: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace @@ -64,20 +67,20 @@ this example doesn't really give a feeling why one would ever want to use Cython, so lets create a more realistic example. -:mod:`pyximport`: Cython Compilation the Easy Way -================================================== +:mod:`pyximport`: Cython Compilation for Developers +--------------------------------------------------- If your module doesn't require any extra C libraries or a special -build setup, then you can use the pyximport module by Paul Prescod and -Stefan Behnel to load .pyx files directly on import, without having to -write a :file:`setup.py` file. It is shipped and installed with -Cython and can be used like this:: +build setup, then you can use the pyximport module, originally developed +by Paul Prescod, to load .pyx files directly on import, without having +to run your :file:`setup.py` file each time you change your code. +It is shipped and installed with Cython and can be used like this:: >>> import pyximport; pyximport.install() >>> import helloworld Hello World -Since Cython 0.11, the :mod:`pyximport` module also has experimental +The :ref:`Pyximport` module also has experimental compilation support for normal Python modules. This allows you to automatically run Cython on every .pyx and .py module that Python imports, including the standard library and installed packages. @@ -85,26 +88,36 @@ case the import mechanism will fall back to loading the Python source modules instead. The .py import mechanism is installed like this:: - >>> pyximport.install(pyimport = True) + >>> pyximport.install(pyimport=True) + +Note that it is not recommended to let :ref:`Pyximport` build code +on end user side as it hooks into their import system. The best way +to cater for end users is to provide pre-built binary packages in the +`wheel `_ packaging format. Fibonacci Fun ============== From the official Python tutorial a simple fibonacci function is defined as: -.. literalinclude:: ../../examples/tutorial/fib1/fib.pyx +.. literalinclude:: ../../examples/tutorial/cython_tutorial/fib.pyx Now following the steps for the Hello World example we first rename the file to have a `.pyx` extension, lets say :file:`fib.pyx`, then we create the :file:`setup.py` file. Using the file created for the Hello World example, all that you need to change is the name of the Cython filename, and the resulting -module name, doing this we have: +module name, doing this we have:: -.. literalinclude:: ../../examples/tutorial/fib1/setup.py + from setuptools import setup + from Cython.Build import cythonize + + setup( + ext_modules=cythonize("fib.pyx"), + ) Build the extension with the same command used for the helloworld.pyx: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace @@ -114,44 +127,178 @@ >>> fib.fib(2000) 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 +.. _primes: + Primes ======= Here's a small example showing some of what can be done. It's a routine for finding prime numbers. You tell it how many primes you want, and it returns them as a Python list. - -:file:`primes.pyx`: - -.. literalinclude:: ../../examples/tutorial/primes/primes.pyx - :linenos: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :linenos: + :caption: primes.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :linenos: + :caption: primes.pyx You'll see that it starts out just like a normal Python function definition, -except that the parameter ``kmax`` is declared to be of type ``int`` . This +except that the parameter ``nb_primes`` is declared to be of type ``int``. This means that the object passed will be converted to a C integer (or a ``TypeError.`` will be raised if it can't be). -Lines 2 and 3 use the ``cdef`` statement to define some local C variables. -Line 4 creates a Python list which will be used to return the result. You'll -notice that this is done exactly the same way it would be in Python. Because -the variable result hasn't been given a type, it is assumed to hold a Python -object. - -Lines 7-9 set up for a loop which will test candidate numbers for primeness -until the required number of primes has been found. Lines 11-12, which try -dividing a candidate by all the primes found so far, are of particular -interest. Because no Python objects are referred to, the loop is translated -entirely into C code, and thus runs very fast. - -When a prime is found, lines 14-15 add it to the p array for fast access by -the testing loop, and line 16 adds it to the result list. Again, you'll notice -that line 16 looks very much like a Python statement, and in fact it is, with -the twist that the C parameter ``n`` is automatically converted to a Python -object before being passed to the append method. Finally, at line 18, a normal -Python return statement returns the result list. +Now, let's dig into the core of the function: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 2,3 + :dedent: + :lineno-start: 2 + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 11,12 + :dedent: + :lineno-start: 11 + + Lines 2, 3, 11 and 12 use the variable annotations + to define some local C variables. + The result is stored in the C array ``p`` during processing, + and will be copied into a Python list at the end (line 26). + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 2,3 + :dedent: + :lineno-start: 2 + + Lines 2 and 3 use the ``cdef`` statement to define some local C variables. + The result is stored in the C array ``p`` during processing, + and will be copied into a Python list at the end (line 26). + +.. NOTE:: You cannot create very large arrays in this manner, because + they are allocated on the C function call :term:`stack`, + which is a rather precious and scarce resource. + To request larger arrays, + or even arrays with a length only known at runtime, + you can learn how to make efficient use of + :ref:`C memory allocation `, + :ref:`Python arrays ` + or :ref:`NumPy arrays ` with Cython. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 5,6 + :dedent: + :lineno-start: 5 + +As in C, declaring a static array requires knowing the size at compile time. +We make sure the user doesn't set a value above 1000 (or we would have a +segmentation fault, just like in C) + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 8,9 + :dedent: + :lineno-start: 8 + + When we run this code from Python, we have to initialize the items in the array. + This is most easily done by filling it with zeros (as seen on line 8-9). + When we compile this with Cython, on the other hand, the array will + behave as in C. It is allocated on the function call stack with a fixed + length of 1000 items that contain arbitrary data from the last time that + memory was used. We will then overwrite those items in our calculation. + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 10-13 + :dedent: + :lineno-start: 10 + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 10-13 + :dedent: + :lineno-start: 10 + +Lines 11-13 set up a while loop which will test numbers-candidates to primes +until the required number of primes has been found. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 14-17 + :dedent: + :lineno-start: 14 + +Lines 15-16, which try to divide a candidate by all the primes found so far, +are of particular interest. Because no Python objects are referred to, +the loop is translated entirely into C code, and thus runs very fast. +You will notice the way we iterate over the ``p`` C array. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 15 + :dedent: + :lineno-start: 15 + +The loop gets translated into a fast C loop and works just like iterating +over a Python list or NumPy array. If you don't slice the C array with +``[:len_p]``, then Cython will loop over the 1000 elements of the array. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 19-23 + :dedent: + :lineno-start: 19 + +If no breaks occurred, it means that we found a prime, and the block of code +after the ``else`` line 20 will be executed. We add the prime found to ``p``. +If you find having an ``else`` after a for-loop strange, just know that it's a +lesser known features of the Python language, and that Cython executes it at +C speed for you. +If the for-else syntax confuses you, see this excellent +`blog post `_. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 25-27 + :dedent: + :lineno-start: 25 + +In line 26, before returning the result, we need to copy our C array into a +Python list, because Python can't read C arrays. Cython can automatically +convert many C types from and to Python types, as described in the +documentation on :ref:`type conversion `, so we can use +a simple list comprehension here to copy the C ``int`` values into a Python +list of Python ``int`` objects, which Cython creates automatically along the way. +You could also have iterated manually over the C array and used +``result_as_list.append(prime)``, the result would have been the same. + +You'll notice we declare a Python list exactly the same way it would be in Python. +Because the variable ``result_as_list`` hasn't been explicitly declared with a type, +it is assumed to hold a Python object, and from the assignment, Cython also knows +that the exact type is a Python list. + +Finally, at line 27, a normal Python return statement returns the result list. + +.. tabs:: + .. group-tab:: Pure Python + + Compiling primes.py with the Cython compiler produces an extension module + which we can try out in the interactive interpreter as follows: -Compiling primes.pyx with the Cython compiler produces an extension module -which we can try out in the interactive interpreter as follows:: + .. group-tab:: Cython + + Compiling primes.pyx with the Cython compiler produces an extension module + which we can try out in the interactive interpreter as follows: + +.. code-block:: python >>> import primes >>> primes.primes(10) @@ -160,10 +307,163 @@ See, it works! And if you're curious about how much work Cython has saved you, take a look at the C code generated for this module. +Cython has a way to visualise where interaction with Python objects and +Python's C-API is taking place. For this, pass the +``annotate=True`` parameter to ``cythonize()``. It produces a HTML file. Let's see: + +.. tabs:: + .. group-tab:: Pure Python + + .. figure:: htmlreport_py.png + :scale: 90 % + + .. group-tab:: Cython + + .. figure:: htmlreport_pyx.png + :scale: 90 % + +If a line is white, it means that the code generated doesn't interact +with Python, so will run as fast as normal C code. The darker the yellow, the more +Python interaction there is in that line. Those yellow lines will usually operate +on Python objects, raise exceptions, or do other kinds of higher-level operations +than what can easily be translated into simple and fast C code. +The function declaration and return use the Python interpreter so it makes +sense for those lines to be yellow. Same for the list comprehension because +it involves the creation of a Python object. But the line ``if n % i == 0:``, why? +We can examine the generated C code to understand: + +.. figure:: python_division.png + +We can see that some checks happen. Because Cython defaults to the +Python behavior, the language will perform division checks at runtime, +just like Python does. You can deactivate those checks by using the +:ref:`compiler directives`. + +Now let's see if we get a speed increase even if there is a division check. +Let's write the same program, but in Python: + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_python.py + :caption: primes_python.py / primes_python_compiled.py + +It is possible to take a plain (unannotated) ``.py`` file and to compile it with Cython. +Let's create a copy of ``primes_python`` and name it ``primes_python_compiled`` +to be able to compare it to the (non-compiled) Python module. +Then we compile that file with Cython, without changing the code. +Now the ``setup.py`` looks like this: + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + from setuptools import setup + from Cython.Build import cythonize + + setup( + ext_modules=cythonize( + ['primes.py', # Cython code file with primes() function + 'primes_python_compiled.py'], # Python code file with primes() function + annotate=True), # enables generation of the html annotation file + ) + + .. group-tab:: Cython + + .. code-block:: python + + from setuptools import setup + from Cython.Build import cythonize + + setup( + ext_modules=cythonize( + ['primes.pyx', # Cython code file with primes() function + 'primes_python_compiled.py'], # Python code file with primes() function + annotate=True), # enables generation of the html annotation file + ) + +Now we can ensure that those two programs output the same values:: + + >>> import primes, primes_python, primes_python_compiled + >>> primes_python.primes(1000) == primes.primes(1000) + True + >>> primes_python_compiled.primes(1000) == primes.primes(1000) + True + +It's possible to compare the speed now:: + + python -m timeit -s 'from primes_python import primes' 'primes(1000)' + 10 loops, best of 3: 23 msec per loop + + python -m timeit -s 'from primes_python_compiled import primes' 'primes(1000)' + 100 loops, best of 3: 11.9 msec per loop + + python -m timeit -s 'from primes import primes' 'primes(1000)' + 1000 loops, best of 3: 1.65 msec per loop + +The cythonize version of ``primes_python`` is 2 times faster than the Python one, +without changing a single line of code. +The Cython version is 13 times faster than the Python version! What could explain this? + +Multiple things: + * In this program, very little computation happen at each line. + So the overhead of the python interpreter is very important. It would be + very different if you were to do a lot computation at each line. Using NumPy for + example. + * Data locality. It's likely that a lot more can fit in CPU cache when using C than + when using Python. Because everything in python is an object, and every object is + implemented as a dictionary, this is not very cache friendly. + +Usually the speedups are between 2x to 1000x. It depends on how much you call +the Python interpreter. As always, remember to profile before adding types +everywhere. Adding types makes your code less readable, so use them with +moderation. + + +Primes with C++ +=============== + +With Cython, it is also possible to take advantage of the C++ language, notably, +part of the C++ standard library is directly importable from Cython code. + +Let's see what our code becomes when using +`vector `_ +from the C++ standard library. + +.. note:: + + Vector in C++ is a data structure which implements a list or stack based + on a resizeable C array. It is similar to the Python ``array`` + type in the ``array`` standard library module. + There is a method `reserve` available which will avoid copies if you know in advance + how many elements you are going to put in the vector. For more details + see `this page from cppreference `_. + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_cpp.py + :linenos: + + .. include:: + ../cimport-warning + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_cpp.pyx + :linenos: + +The first line is a compiler directive. It tells Cython to compile your code to C++. +This will enable the use of C++ language features and the C++ standard library. +Note that it isn't possible to compile Cython code to C++ with `pyximport`. You +should use a :file:`setup.py` or a notebook to run this example. + +You can see that the API of a vector is similar to the API of a Python list, +and can sometimes be used as a drop-in replacement in Cython. + +For more details about using C++ with Cython, see :ref:`wrapping-cplusplus`. + Language Details ================ For more about the Cython language, see :ref:`language-basics`. To dive right in to using Cython in a numerical computation context, -see :ref:`numpy_tutorial`. - +see :ref:`memoryviews`. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/data.py cython-0.20.1+1~202203241016-9537/docs/src/tutorial/data.py --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/data.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/data.py 2022-03-24 10:16:46.000000000 +0000 @@ -2,11 +2,11 @@ 'title': 'Cython Tutorial', 'paper_abstract': ''' Cython is a programming language based on Python with extra -syntax to provide static type declarations. This takes advantage of the -benefits of Python while allowing one to achieve the speed of C. -In this paper we describe the Cython language and show how it can -be used both to write optimized code and to interface with external -C libraries. +syntax to provide static type declarations. This takes advantage of the +benefits of Python while allowing one to achieve the speed of C. +In this paper we describe the Cython language and show how it can +be used both to write optimized code and to interface with external +C libraries. ''', 'authors': [ {'first_names': 'Stefan', diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/embedding.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/embedding.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/embedding.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/embedding.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,77 @@ +.. highlight:: cython + +.. _embedding: + +********************************************** +Embedding Cython modules in C/C++ applications +********************************************** + +**This is a stub documentation page. PRs very welcome.** + +Quick links: + +* `CPython docs `_ + +* `Cython Wiki `_ + +* See the ``--embed`` option to the ``cython`` and ``cythonize`` frontends + for generating a C main function and the + `cython_freeze `_ + script for merging multiple extension modules into one library. + +* `Embedding demo program `_ + +* See the documentation of the `module init function + `_ + in CPython and `PEP 489 `_ regarding the module + initialisation mechanism in CPython 3.5 and later. + + +Initialising your main module +============================= + +Most importantly, DO NOT call the module init function instead of importing +the module. This is not the right way to initialise an extension module. +(It was always wrong but used to work before, but since Python 3.5, it is +wrong *and* no longer works.) + +For details, see the documentation of the +`module init function `_ +in CPython and `PEP 489 `_ regarding the module +initialisation mechanism in CPython 3.5 and later. + +The `PyImport_AppendInittab() `_ +function in CPython allows registering statically (or dynamically) linked extension +modules for later imports. An example is given in the documentation of the module +init function that is linked above. + + +Embedding example code +====================== + +The following is a simple example that shows the main steps for embedding a +Cython module (``embedded.pyx``) in Python 3.x. + +First, here is a Cython module that exports a C function to be called by external +code. Note that the ``say_hello_from_python()`` function is declared as ``public`` +to export it as a linker symbol that can be used by other C files, which in this +case is ``embedded_main.c``. + +.. literalinclude:: ../../examples/tutorial/embedding/embedded.pyx + +The C ``main()`` function of your program could look like this: + +.. literalinclude:: ../../examples/tutorial/embedding/embedded_main.c + :linenos: + :language: c + +(Adapted from the `CPython documentation +`_.) + +Instead of writing such a ``main()`` function yourself, you can also let +Cython generate one into your module's C file with the ``cython --embed`` +option. Or use the +`cython_freeze `_ +script to embed multiple modules. See the +`embedding demo program `_ +for a complete example setup. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/external.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/external.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/external.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/external.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,9 @@ Calling C functions ==================== +.. include:: + ../two-syntax-variants-used + This tutorial describes shortly what you need to know in order to call C library functions from Cython code. For a longer and more comprehensive tutorial about using external C libraries, wrapping them @@ -13,13 +16,19 @@ For example, let's say you need a low-level way to parse a number from a ``char*`` value. You could use the ``atoi()`` function, as defined -by the ``stdlib.h`` header file. This can be done as follows:: +by the ``stdlib.h`` header file. This can be done as follows: + +.. tabs:: + + .. group-tab:: Pure Python - from libc.stdlib cimport atoi + .. literalinclude:: ../../examples/tutorial/external/atoi.py + :caption: atoi.py - cdef parse_charptr_to_py_int(char* s): - assert s is not NULL, "byte string value is NULL" - return atoi(s) # note: atoi() has no error detection! + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/atoi.pyx + :caption: atoi.pyx You can find a complete list of these standard cimport files in Cython's source package @@ -30,20 +39,35 @@ Cython also has a complete set of declarations for CPython's C-API. For example, to test at C compilation time which CPython version -your code is being compiled with, you can do this:: +your code is being compiled with, you can do this: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/py_version_hex.py + :caption: py_version_hex.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/py_version_hex.pyx + :caption: py_version_hex.pyx - from cpython.version cimport PY_VERSION_HEX +.. _libc.math: - # Python version >= 3.2 final ? - print PY_VERSION_HEX >= 0x030200F0 +Cython also provides declarations for the C math library: -Cython also provides declarations for the C math library:: +.. tabs:: - from libc.math cimport sin + .. group-tab:: Pure Python - cdef double f(double x): - return sin(x*x) + .. literalinclude:: ../../examples/tutorial/external/libc_sin.py + :caption: libc_sin.py + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/libc_sin.pyx + :caption: libc_sin.pyx Dynamic linking --------------- @@ -51,25 +75,10 @@ The libc math library is special in that it is not linked by default on some Unix-like systems, such as Linux. In addition to cimporting the declarations, you must configure your build system to link against the -shared library ``m``. For distutils, it is enough to add it to the -``libraries`` parameter of the ``Extension()`` setup:: - - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize - - ext_modules=[ - Extension("demo", - sources=["demo.pyx"], - libraries=["m"] # Unix-like specific - ) - ] - - setup( - name = "Demos", - ext_modules = cythonize(ext_modules) - ) +shared library ``m``. For setuptools, it is enough to add it to the +``libraries`` parameter of the ``Extension()`` setup: +.. literalinclude:: ../../examples/tutorial/external/setup.py External declarations --------------------- @@ -95,15 +104,9 @@ Note that you can easily export an external C function from your Cython module by declaring it as ``cpdef``. This generates a Python wrapper for it and adds it to the module dict. Here is a Cython module that -provides direct access to the C ``sin()`` function for Python code:: +provides direct access to the C ``sin()`` function for Python code: - """ - >>> sin(0) - 0.0 - """ - - cdef extern from "math.h": - cpdef double sin(double x) +.. literalinclude:: ../../examples/tutorial/external/cpdef_sin.pyx You get the same result when this declaration appears in the ``.pxd`` file that belongs to the Cython module (i.e. that has the same name, @@ -112,6 +115,9 @@ while still providing an automatically generated Python wrapper in this specific module. +.. note:: External declarations must be placed in a ``.pxd`` file in Pure + Python mode. + Naming parameters ----------------- @@ -123,20 +129,28 @@ char* strstr(const char*, const char*) However, this prevents Cython code from calling it with keyword -arguments (supported since Cython 0.19). It is therefore preferable -to write the declaration like this instead:: +arguments. It is therefore preferable +to write the declaration like this instead: - cdef extern from "string.h": - char* strstr(const char *haystack, const char *needle) +.. literalinclude:: ../../examples/tutorial/external/keyword_args.pyx You can now make it clear which of the two arguments does what in your call, thus avoiding any ambiguities and often making your code -more readable:: +more readable: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/keyword_args_call.py + :caption: keyword_args_call.py + .. literalinclude:: ../../examples/tutorial/external/strstr.pxd + :caption: strstr.pxd - cdef char* data = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh" + .. group-tab:: Cython - pos = strstr(needle='akd', haystack=data) - print pos != NULL + .. literalinclude:: ../../examples/tutorial/external/keyword_args_call.pyx + :caption: keyword_args_call.pyx Note that changing existing parameter names later is a backwards incompatible API modification, just as for Python code. Thus, if Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/tutorial/htmlreport_py.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/tutorial/htmlreport_py.png differ Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/tutorial/htmlreport_pyx.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/tutorial/htmlreport_pyx.png differ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/index.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/index.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/index.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/index.rst 2022-03-24 10:16:46.000000000 +0000 @@ -13,10 +13,10 @@ profiling_tutorial strings memory_allocation + embedding pure numpy array readings related_work appendix - diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/memory_allocation.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/memory_allocation.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/memory_allocation.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/memory_allocation.rst 2022-03-24 10:16:46.000000000 +0000 @@ -4,6 +4,9 @@ Memory Allocation ***************** +.. include:: + ../two-syntax-variants-used + Dynamic memory allocation is mostly a non-issue in Python. Everything is an object, and the reference counting system and garbage collector automatically return memory to the system when it is no longer being used. @@ -19,10 +22,10 @@ amount of overhead, which can then makes a case for doing manual memory management in C. -Simple C values and structs (such as a local variable ``cdef double x``) are -usually allocated on the stack and passed by value, but for larger and more +Simple C values and structs (such as a local variable ``cdef double x`` / ``x: cython.double``) are +usually :term:`allocated on the stack` and passed by value, but for larger and more complicated objects (e.g. a dynamically-sized list of doubles), the memory must -be manually requested and released. C provides the functions :c:func:`malloc`, +be :term:`manually requested and released`. C provides the functions :c:func:`malloc`, :c:func:`realloc`, and :c:func:`free` for this purpose, which can be imported in cython from ``clibc.stdlib``. Their signatures are: @@ -32,27 +35,17 @@ void* realloc(void* ptr, size_t size) void free(void* ptr) -A very simple example of malloc usage is the following:: +A very simple example of malloc usage is the following: + + +.. tabs:: + .. group-tab:: Pure Python - import random - from libc.stdlib cimport malloc, free + .. literalinclude:: ../../examples/tutorial/memory_allocation/malloc.py - def random_noise(int number=1): - cdef int i - # allocate number * sizeof(double) bytes of memory - cdef double *my_array = malloc(number * sizeof(double)) - if not my_array: - raise MemoryError() - - try: - ran = random.normalvariate - for i in range(number): - my_array[i] = ran(0,1) - - return [ my_array[i] for i in range(number) ] - finally: - # return the previously allocated memory to the system - free(my_array) + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/memory_allocation/malloc.pyx Note that the C-API functions for allocating memory on the Python heap are generally preferred over the low-level C functions above as the @@ -62,9 +55,20 @@ costly operating system calls. The C-API functions can be found in the ``cpython.mem`` standard -declarations file:: +declarations file: + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python + + from cython.cimports.cpython.mem import PyMem_Malloc, PyMem_Realloc, PyMem_Free + + .. group-tab:: Cython + + .. code-block:: cython - from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free Their interface and usage is identical to that of the corresponding low-level C functions. @@ -79,28 +83,13 @@ If a chunk of memory needs a larger lifetime than can be managed by a ``try..finally`` block, another helpful idiom is to tie its lifetime to a Python object to leverage the Python runtime's memory management, -e.g.:: +e.g.: - cdef class SomeMemory: +.. tabs:: + .. group-tab:: Pure Python - cdef double* data + .. literalinclude:: ../../examples/tutorial/memory_allocation/some_memory.py - def __cinit__(self, size_t number): - # allocate some memory (uninitialised, may contain arbitrary data) - self.data = PyMem_Malloc(number * sizeof(double)) - if not self.data: - raise MemoryError() - - def resize(self, size_t new_number): - # Allocates new_number * sizeof(double) bytes, - # preserving the current content and making a best-effort to - # re-use the original data location. - mem = PyMem_Realloc(self.data, new_number * sizeof(double)) - if not mem: - raise MemoryError() - # Only overwrite the pointer if the memory was really reallocated. - # On error (mem is NULL), the originally memory has not been freed. - self.data = mem + .. group-tab:: Cython - def __dealloc__(self): - PyMem_Free(self.data) # no-op if self.data is NULL + .. literalinclude:: ../../examples/tutorial/memory_allocation/some_memory.pyx diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/numpy.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/numpy.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/numpy.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/numpy.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,15 @@ +.. _working-numpy: + ======================= Working with NumPy ======================= +.. NOTE:: Cython 0.16 introduced typed memoryviews as a successor to the NumPy + integration described here. They are easier to use than the buffer syntax + below, have less overhead, and can be passed around without requiring the GIL. + They should be preferred to the syntax presented in this page. + See :ref:`Cython for NumPy users `. + You can use NumPy from Cython exactly the same as in regular Python, but by doing so you are losing potentially high speedups because Cython has support for fast access to NumPy arrays. Let's see how this works with a simple @@ -13,56 +21,14 @@ :file:`convolve_py.py` for the Python version and :file:`convolve1.pyx` for the Cython version -- Cython uses ".pyx" as its file suffix. -.. code-block:: python - - from __future__ import division - import numpy as np - def naive_convolve(f, g): - # f is an image and is indexed by (v, w) - # g is a filter kernel and is indexed by (s, t), - # it needs odd dimensions - # h is the output image and is indexed by (x, y), - # it is not cropped - if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: - raise ValueError("Only odd dimensions on filter supported") - # smid and tmid are number of pixels between the center pixel - # and the edge, ie for a 5x5 filter they will be 2. - # - # The output size is calculated by adding smid, tmid to each - # side of the dimensions of the input image. - vmax = f.shape[0] - wmax = f.shape[1] - smax = g.shape[0] - tmax = g.shape[1] - smid = smax // 2 - tmid = tmax // 2 - xmax = vmax + 2*smid - ymax = wmax + 2*tmid - # Allocate result image. - h = np.zeros([xmax, ymax], dtype=f.dtype) - # Do convolution - for x in range(xmax): - for y in range(ymax): - # Calculate pixel value for h at (x,y). Sum one component - # for each pixel (s, t) of the filter g. - s_from = max(smid - x, -smid) - s_to = min((xmax - x) - smid, smid + 1) - t_from = max(tmid - y, -tmid) - t_to = min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = x - smid + s - w = y - tmid + t - value += g[smid - s, tmid - t] * f[v, w] - h[x, y] = value - return h +.. literalinclude:: ../../examples/tutorial/numpy/convolve_py.py -This should be compiled to produce :file:`yourmod.so` (for Linux systems). We +This should be compiled to produce :file:`yourmod.so` (for Linux systems, on Windows +systems, it will be :file:`yourmod.pyd`). We run a Python session to test both the Python version (imported from ``.py``-file) and the compiled Cython module. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [1]: import numpy as np In [2]: import convolve_py @@ -97,81 +63,13 @@ ============= To add types we use custom Cython syntax, so we are now breaking Python source -compatibility. Consider this code (*read the comments!*) :: +compatibility. Consider this code (*read the comments!*) : - from __future__ import division - import numpy as np - # "cimport" is used to import special compile-time information - # about the numpy module (this is stored in a file numpy.pxd which is - # currently part of the Cython distribution). - cimport numpy as np - # We now need to fix a datatype for our arrays. I've used the variable - # DTYPE for this, which is assigned to the usual NumPy runtime - # type info object. - DTYPE = np.int - # "ctypedef" assigns a corresponding compile-time type to DTYPE_t. For - # every type in the numpy module there's a corresponding compile-time - # type with a _t-suffix. - ctypedef np.int_t DTYPE_t - # "def" can type its arguments but not have a return type. The type of the - # arguments for a "def" function is checked at run-time when entering the - # function. - # - # The arrays f, g and h is typed as "np.ndarray" instances. The only effect - # this has is to a) insert checks that the function arguments really are - # NumPy arrays, and b) make some attribute access like f.shape[0] much - # more efficient. (In this example this doesn't matter though.) - def naive_convolve(np.ndarray f, np.ndarray g): - if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: - raise ValueError("Only odd dimensions on filter supported") - assert f.dtype == DTYPE and g.dtype == DTYPE - # The "cdef" keyword is also used within functions to type variables. It - # can only be used at the top indentation level (there are non-trivial - # problems with allowing them in other places, though we'd love to see - # good and thought out proposals for it). - # - # For the indices, the "int" type is used. This corresponds to a C int, - # other C types (like "unsigned int") could have been used instead. - # Purists could use "Py_ssize_t" which is the proper Python type for - # array indices. - cdef int vmax = f.shape[0] - cdef int wmax = f.shape[1] - cdef int smax = g.shape[0] - cdef int tmax = g.shape[1] - cdef int smid = smax // 2 - cdef int tmid = tmax // 2 - cdef int xmax = vmax + 2*smid - cdef int ymax = wmax + 2*tmid - cdef np.ndarray h = np.zeros([xmax, ymax], dtype=DTYPE) - cdef int x, y, s, t, v, w - # It is very important to type ALL your variables. You do not get any - # warnings if not, only much slower code (they are implicitly typed as - # Python objects). - cdef int s_from, s_to, t_from, t_to - # For the value variable, we want to use the same data type as is - # stored in the array, so we use "DTYPE_t" as defined above. - # NB! An important side-effect of this is that if "value" overflows its - # datatype size, it will simply wrap around like in C, rather than raise - # an error like in Python. - cdef DTYPE_t value - for x in range(xmax): - for y in range(ymax): - s_from = max(smid - x, -smid) - s_to = min((xmax - x) - smid, smid + 1) - t_from = max(tmid - y, -tmid) - t_to = min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = x - smid + s - w = y - tmid + t - value += g[smid - s, tmid - t] * f[v, w] - h[x, y] = value - return h +.. literalinclude:: ../../examples/tutorial/numpy/convolve2.pyx After building this and continuing my (very informal) benchmarks, I get: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [21]: import convolve2 In [22]: %timeit -n2 -r3 convolve2.naive_convolve(f, g) @@ -199,7 +97,7 @@ Usage: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [18]: import convolve3 In [19]: %timeit -n3 -r100 convolve3.naive_convolve(f, g) @@ -245,7 +143,7 @@ The function call overhead now starts to play a role, so we compare the latter two examples with larger N: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [11]: %timeit -n3 -r100 convolve4.naive_convolve(f, g) 3 loops, best of 100: 5.97 ms per loop @@ -272,6 +170,16 @@ The actual rules are a bit more complicated but the main message is clear: Do not use typed objects without knowing that they are not set to None. +What typing does not do +======================= + +The main purpose of typing things as :obj:`ndarray` is to allow efficient +indexing of single elements, and to speed up access to a small number of +attributes such as ``.shape``. Typing does not allow Cython to speed +up mathematical operations on the whole array (for example, adding two arrays +together). Typing does not allow Cython to speed up calls to Numpy global +functions or to methods of the array. + More generic code ================== diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/profiling_tutorial.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/profiling_tutorial.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/profiling_tutorial.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/profiling_tutorial.rst 2022-03-24 10:16:46.000000000 +0000 @@ -6,7 +6,10 @@ Profiling ********* -This part describes the profiling abilities of Cython. If you are familiar +.. include:: + ../two-syntax-variants-used + +This part describes the profiling abilities of Cython. If you are familiar with profiling pure Python code, you can only read the first section (:ref:`profiling_basics`). If you are not familiar with Python profiling you should also read the tutorial (:ref:`profiling_tutorial`) which takes you @@ -17,7 +20,7 @@ Cython Profiling Basics ======================= -Profiling in Cython is controlled by a compiler directive. +Profiling in Cython is controlled by a compiler directive. It can be set either for an entire file or on a per function basis via a Cython decorator. @@ -26,7 +29,7 @@ Profiling is enabled for a complete source file via a global directive to the Cython compiler at the top of a file:: - + # cython: profile=True Note that profiling gives a slight overhead to each function call therefore making @@ -35,7 +38,7 @@ Once enabled, your Cython code will behave just like Python code when called from the cProfile module. This means you can just profile your Cython code -together with your Python code using the same tools as for Python code alone. +together with your Python code using the same tools as for Python code alone. Disabling profiling function wise --------------------------------- @@ -44,14 +47,17 @@ functions that you rather do not want to see in your profile - either because you plan to inline them anyway or because you are sure that you can't make them any faster - you can use a special decorator to disable profiling for one -function only:: +function only (regardless of whether it is globally enabled or not): + +.. tabs:: + + .. group-tab:: Pure Python - cimport cython + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/often_called.py - @cython.profile(False) - def my_often_called_function(): - pass + .. group-tab:: Cython + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/often_called.pyx Enabling line tracing --------------------- @@ -80,8 +86,8 @@ -------------------------- Since Cython 0.23, line tracing (see above) also enables support for coverage -reporting with the `coverage.py `_ tool. -To make the coverage analysis understand Cython modules, you also need to enable +reporting with the `coverage.py `_ tool. To +make the coverage analysis understand Cython modules, you also need to enable Cython's coverage plugin in your ``.coveragerc`` file as follows: .. code-block:: ini @@ -111,35 +117,25 @@ ================== This will be a complete tutorial, start to finish, of profiling Python code, -turning it into Cython code and keep profiling until it is fast enough. +turning it into Cython code and keep profiling until it is fast enough. As a toy example, we would like to evaluate the summation of the reciprocals of squares up to a certain integer :math:`n` for evaluating :math:`\pi`. The relation we want to use has been proven by Euler in 1735 and is known as the -`Basel problem `_. +`Basel problem `_. .. math:: - \pi^2 = 6 \sum_{k=1}^{\infty} \frac{1}{k^2} = - 6 \lim_{k \to \infty} \big( \frac{1}{1^2} + + \pi^2 = 6 \sum_{k=1}^{\infty} \frac{1}{k^2} = + 6 \lim_{k \to \infty} \big( \frac{1}{1^2} + \frac{1}{2^2} + \dots + \frac{1}{k^2} \big) \approx 6 \big( \frac{1}{1^2} + \frac{1}{2^2} + \dots + \frac{1}{n^2} \big) -A simple Python code for evaluating the truncated sum looks like this:: +A simple Python code for evaluating the truncated sum looks like this: + +.. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi.py + :caption: calc_pi.py - #!/usr/bin/env python - # encoding: utf-8 - # filename: calc_pi.py - - def recip_square(i): - return 1./i**2 - - def approx_pi(n=10000000): - val = 0. - for k in range(1,n+1): - val += recip_square(k) - return (6 * val)**.5 - On my box, this needs approximately 4 seconds to run the function with the default n. The higher we choose n, the better will be the approximation for :math:`\pi`. An experienced Python programmer will already see plenty of @@ -147,24 +143,14 @@ Never optimize without having profiled. Let me repeat this: **Never** optimize without having profiled your code. Your thoughts about which part of your code takes too much time are wrong. At least, mine are always wrong. So let's -write a short script to profile our code:: - - #!/usr/bin/env python - # encoding: utf-8 - # filename: profile.py - - import pstats, cProfile +write a short script to profile our code: - import calc_pi +.. literalinclude:: ../../examples/tutorial/profiling_tutorial/profile.py + :caption: profile.py - cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") +Running this on my box gives the following output: - s = pstats.Stats("Profile.prof") - s.strip_dirs().sort_stats("time").print_stats() - -Running this on my box gives the following output:: - - TODO: how to display this not as code but verbatim? +.. code-block:: none Sat Nov 7 17:40:54 2009 Profile.prof @@ -173,8 +159,8 @@ Ordered by: internal time ncalls tottime percall cumtime percall filename:lineno(function) - 1 3.243 3.243 6.211 6.211 calc_pi.py:7(approx_pi) - 10000000 2.526 0.000 2.526 0.000 calc_pi.py:4(recip_square) + 1 3.243 3.243 6.211 6.211 calc_pi.py:4(approx_pi) + 10000000 2.526 0.000 2.526 0.000 calc_pi.py:1(recip_square) 1 0.442 0.442 0.442 0.442 {range} 1 0.000 0.000 6.211 6.211 :1() 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} @@ -182,171 +168,219 @@ This contains the information that the code runs in 6.2 CPU seconds. Note that the code got slower by 2 seconds because it ran inside the cProfile module. The table contains the real valuable information. You might want to check the -Python `profiling documentation `_ +Python `profiling documentation `_ for the nitty gritty details. The most important columns here are totime (total time spent in this function **not** counting functions that were called by this function) and cumtime (total time spent in this function **also** counting the -functions called by this function). Looking at the tottime column, we see that -approximately half the time is spent in approx_pi and the other half is spent -in recip_square. Also half a second is spent in range ... of course we should +functions called by this function). Looking at the tottime column, we see that +approximately half the time is spent in ``approx_pi()`` and the other half is spent +in ``recip_square()``. Also half a second is spent in range ... of course we should have used xrange for such a big iteration. And in fact, just changing range to xrange makes the code run in 5.8 seconds. We could optimize a lot in the pure Python version, but since we are interested in Cython, let's move forward and bring this module to Cython. We would do this -anyway at some time to get the loop run faster. Here is our first Cython version:: +anyway at some time to get the loop run faster. Here is our first Cython version: - # encoding: utf-8 - # cython: profile=True - # filename: calc_pi.pyx +.. tabs:: + + .. group-tab:: Pure Python - def recip_square(int i): - return 1./i**2 + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_2.py + :caption: calc_pi.py - def approx_pi(int n=10000000): - cdef double val = 0. - cdef int k - for k in xrange(1,n+1): - val += recip_square(k) - return (6 * val)**.5 + .. group-tab:: Cython -Note the second line: We have to tell Cython that profiling should be enabled. + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_2.pyx + :caption: calc_pi.pyx + +Note the first line: We have to tell Cython that profiling should be enabled. This makes the Cython code slightly slower, but without this we would not get meaningful output from the cProfile module. The rest of the code is mostly -unchanged, I only typed some variables which will likely speed things up a bit. +unchanged, I only typed some variables which will likely speed things up a bit. We also need to modify our profiling script to import the Cython module directly. -Here is the complete version adding the import of the pyximport module:: +Here is the complete version adding the import of the :ref:`Pyximport` module: + +.. literalinclude:: ../../examples/tutorial/profiling_tutorial/profile_2.py + :caption: profile.py + +We only added two lines, the rest stays completely the same. Alternatively, we could also +manually compile our code into an extension; we wouldn't need to change the +profile script then at all. The script now outputs the following: - #!/usr/bin/env python - # encoding: utf-8 - # filename: profile.py +.. tabs:: - import pstats, cProfile + .. group-tab:: Pure Python - import pyximport - pyximport.install() + .. code-block:: none - import calc_pi + Sat Nov 7 18:02:33 2009 Profile.prof - cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") + 10000004 function calls in 4.406 CPU seconds - s = pstats.Stats("Profile.prof") - s.strip_dirs().sort_stats("time").print_stats() + Ordered by: internal time -We only added two lines, the rest stays completely the same. Alternatively, we could also -manually compile our code into an extension; we wouldn't need to change the -profile script then at all. The script now outputs the following:: + ncalls tottime percall cumtime percall filename:lineno(function) + 1 3.305 3.305 4.406 4.406 calc_pi.py:6(approx_pi) + 10000000 1.101 0.000 1.101 0.000 calc_pi.py:3(recip_square) + 1 0.000 0.000 4.406 4.406 {calc_pi.approx_pi} + 1 0.000 0.000 4.406 4.406 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - Sat Nov 7 18:02:33 2009 Profile.prof + .. group-tab:: Cython - 10000004 function calls in 4.406 CPU seconds + .. code-block:: none - Ordered by: internal time + Sat Nov 7 18:02:33 2009 Profile.prof - ncalls tottime percall cumtime percall filename:lineno(function) - 1 3.305 3.305 4.406 4.406 calc_pi.pyx:7(approx_pi) - 10000000 1.101 0.000 1.101 0.000 calc_pi.pyx:4(recip_square) - 1 0.000 0.000 4.406 4.406 {calc_pi.approx_pi} - 1 0.000 0.000 4.406 4.406 :1() - 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + 10000004 function calls in 4.406 CPU seconds -We gained 1.8 seconds. Not too shabby. Comparing the output to the previous, we -see that recip_square function got faster while the approx_pi function has not -changed a lot. Let's concentrate on the recip_square function a bit more. First -note, that this function is not to be called from code outside of our module; -so it would be wise to turn it into a cdef to reduce call overhead. We should -also get rid of the power operator: it is turned into a pow(i,2) function call by -Cython, but we could instead just write i*i which could be faster. The + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 3.305 3.305 4.406 4.406 calc_pi.pyx:6(approx_pi) + 10000000 1.101 0.000 1.101 0.000 calc_pi.pyx:3(recip_square) + 1 0.000 0.000 4.406 4.406 {calc_pi.approx_pi} + 1 0.000 0.000 4.406 4.406 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + +We gained 1.8 seconds. Not too shabby. Comparing the output to the previous, we +see that the ``recip_square()`` function got faster while the ``approx_pi()`` +function has not changed a lot. Let's concentrate on the ``recip_square()`` function +a bit more. First, note that this function is not to be called from code outside +of our module; so it would be wise to turn it into a cdef to reduce call overhead. +We should also get rid of the power operator: it is turned into a ``pow(i, 2)`` function +call by Cython, but we could instead just write ``i * i`` which could be faster. The whole function is also a good candidate for inlining. Let's look at the -necessary changes for these ideas:: +necessary changes for these ideas: - # encoding: utf-8 - # cython: profile=True - # filename: calc_pi.pyx +.. tabs:: - cdef inline double recip_square(int i): - return 1./(i*i) + .. group-tab:: Pure Python - def approx_pi(int n=10000000): - cdef double val = 0. - cdef int k - for k in xrange(1,n+1): - val += recip_square(k) - return (6 * val)**.5 + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_3.py + :caption: calc_pi.py -Now running the profile script yields:: + .. group-tab:: Cython - Sat Nov 7 18:10:11 2009 Profile.prof + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_3.pyx + :caption: calc_pi.pyx - 10000004 function calls in 2.622 CPU seconds +Note that the ``except``/``@exceptval`` declaration is needed in the signature of ``recip_square()`` +in order to propagate division by zero errors. - Ordered by: internal time +Now running the profile script yields: - ncalls tottime percall cumtime percall filename:lineno(function) - 1 1.782 1.782 2.622 2.622 calc_pi.pyx:7(approx_pi) - 10000000 0.840 0.000 0.840 0.000 calc_pi.pyx:4(recip_square) - 1 0.000 0.000 2.622 2.622 {calc_pi.approx_pi} - 1 0.000 0.000 2.622 2.622 :1() - 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: none + + Sat Nov 7 18:10:11 2009 Profile.prof + + 10000004 function calls in 2.622 CPU seconds + + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 1.782 1.782 2.622 2.622 calc_pi.py:9(approx_pi) + 10000000 0.840 0.000 0.840 0.000 calc_pi.py:6(recip_square) + 1 0.000 0.000 2.622 2.622 {calc_pi.approx_pi} + 1 0.000 0.000 2.622 2.622 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + .. group-tab:: Cython + + .. code-block:: none + + Sat Nov 7 18:10:11 2009 Profile.prof + + 10000004 function calls in 2.622 CPU seconds + + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 1.782 1.782 2.622 2.622 calc_pi.pyx:9(approx_pi) + 10000000 0.840 0.000 0.840 0.000 calc_pi.pyx:6(recip_square) + 1 0.000 0.000 2.622 2.622 {calc_pi.approx_pi} + 1 0.000 0.000 2.622 2.622 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} That bought us another 1.8 seconds. Not the dramatic change we could have -expected. And why is recip_square still in this table; it is supposed to be +expected. And why is ``recip_square()`` still in this table; it is supposed to be inlined, isn't it? The reason for this is that Cython still generates profiling code even if the function call is eliminated. Let's tell it to not -profile recip_square any more; we couldn't get the function to be much faster anyway:: +profile ``recip_square()`` any more; we couldn't get the function to be much faster anyway: - # encoding: utf-8 - # cython: profile=True - # filename: calc_pi.pyx +.. tabs:: - cimport cython + .. group-tab:: Pure Python - @cython.profile(False) - cdef inline double recip_square(int i): - return 1./(i*i) + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_4.py + :caption: calc_pi.py - def approx_pi(int n=10000000): - cdef double val = 0. - cdef int k - for k in xrange(1,n+1): - val += recip_square(k) - return (6 * val)**.5 + .. group-tab:: Cython -Running this shows an interesting result:: + .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_4.pyx + :caption: calc_pi.pyx - Sat Nov 7 18:15:02 2009 Profile.prof - 4 function calls in 0.089 CPU seconds +Running this shows an interesting result: - Ordered by: internal time +.. tabs:: - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.089 0.089 0.089 0.089 calc_pi.pyx:10(approx_pi) - 1 0.000 0.000 0.089 0.089 {calc_pi.approx_pi} - 1 0.000 0.000 0.089 0.089 :1() - 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + .. group-tab:: Pure Python + + .. code-block:: none + + Sat Nov 7 18:15:02 2009 Profile.prof + + 4 function calls in 0.089 CPU seconds + + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.089 0.089 0.089 0.089 calc_pi.py:12(approx_pi) + 1 0.000 0.000 0.089 0.089 {calc_pi.approx_pi} + 1 0.000 0.000 0.089 0.089 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + .. group-tab:: Cython + + .. code-block:: none + + Sat Nov 7 18:15:02 2009 Profile.prof + + 4 function calls in 0.089 CPU seconds + + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.089 0.089 0.089 0.089 calc_pi.pyx:12(approx_pi) + 1 0.000 0.000 0.089 0.089 {calc_pi.approx_pi} + 1 0.000 0.000 0.089 0.089 :1() + 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} First note the tremendous speed gain: this version only takes 1/50 of the time -of our first Cython version. Also note that recip_square has vanished from the -table like we wanted. But the most peculiar and import change is that -approx_pi also got much faster. This is a problem with all profiling: calling a -function in a profile run adds a certain overhead to the function call. This +of our first Cython version. Also note that ``recip_square()`` has vanished from the +table like we wanted. But the most peculiar and important change is that +``approx_pi()`` also got much faster. This is a problem with all profiling: calling a +function in a profile run adds a certain overhead to the function call. This overhead is **not** added to the time spent in the called function, but to the -time spent in the **calling** function. In this example, approx_pi didn't need 2.622 -seconds in the last run; but it called recip_square 10000000 times, each time taking a -little to set up profiling for it. This adds up to the massive time loss of -around 2.6 seconds. Having disabled profiling for the often called function now -reveals realistic timings for approx_pi; we could continue optimizing it now if +time spent in the **calling** function. In this example, ``approx_pi()`` didn't need 2.622 +seconds in the last run; but it called ``recip_square()`` 10000000 times, each time taking a +little to set up profiling for it. This adds up to the massive time loss of +around 2.6 seconds. Having disabled profiling for the often called function now +reveals realistic timings for ``approx_pi()``; we could continue optimizing it now if needed. This concludes this profiling tutorial. There is still some room for improvement in this code. We could try to replace the power operator in -approx_pi with a call to sqrt from the C stdlib; but this is not necessarily -faster than calling pow(x,0.5). +``approx_pi()`` with a call to sqrt from the C stdlib; but this is not necessarily +faster than calling ``pow(x, 0.5)``. Even so, the result we achieved here is quite satisfactory: we came up with a solution that is much faster then our original Python version while retaining functionality and readability. - - diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/pure.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/pure.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/pure.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/pure.rst 2022-03-24 10:16:46.000000000 +0000 @@ -12,17 +12,22 @@ To go beyond that, Cython provides language constructs to add static typing and cythonic functionalities to a Python module to make it run much faster when compiled, while still allowing it to be interpreted. -This is accomplished either via an augmenting :file:`.pxd` file, or +This is accomplished via an augmenting ``.pxd`` file, via Python +type :ref:`pep484_type_annotations` (following +`PEP 484 `_ and +`PEP 526 `_), and/or via special functions and decorators available after importing the magic -``cython`` module. +``cython`` module. All three ways can be combined at need, although +projects would commonly decide on a specific way to keep the static type +information easy to manage. Although it is not typically recommended over writing straight Cython code in a :file:`.pyx` file, there are legitimate reasons to do this - easier -testing, collaboration with pure Python developers, etc. In pure mode, you -are more or less restricted to code that can be expressed (or at least -emulated) in Python, plus static type declarations. Anything beyond that -can only be done in .pyx files with extended language syntax, because it -depends on features of the Cython compiler. +testing and debugging, collaboration with pure Python developers, etc. +In pure mode, you are more or less restricted to code that can be expressed +(or at least emulated) in Python, plus static type declarations. Anything +beyond that can only be done in .pyx files with extended language syntax, +because it depends on features of the Cython compiler. Augmenting .pxd @@ -42,49 +47,17 @@ being compiled, it will be searched for :keyword:`cdef` classes and :keyword:`cdef`/:keyword:`cpdef` functions and methods. The compiler will then convert the corresponding classes/functions/methods in the :file:`.py` -file to be of the declared type. Thus if one has a file :file:`A.py`:: +file to be of the declared type. Thus if one has a file :file:`A.py`: - def myfunction(x, y=2): - a = x-y - return a + x * y +.. literalinclude:: ../../examples/tutorial/pure/A.py - def _helper(a): - return a + 1 +and adds :file:`A.pxd`: - class A: - def __init__(self, b=0): - self.a = 3 - self.b = b +.. literalinclude:: ../../examples/tutorial/pure/A.pxd - def foo(self, x): - print x + _helper(1.0) +then Cython will compile the :file:`A.py` as if it had been written as follows: -and adds :file:`A.pxd`:: - - cpdef int myfunction(int x, int y=*) - cdef double _helper(double a) - - cdef class A: - cdef public int a,b - cpdef foo(self, double x) - -then Cython will compile the :file:`A.py` as if it had been written as follows:: - - cpdef int myfunction(int x, int y=2): - a = x-y - return a + x * y - - cdef double _helper(double a): - return a + 1 - - cdef class A: - cdef public int a,b - def __init__(self, b=0): - self.a = 3 - self.b = b - - cpdef foo(self, double x): - print x + _helper(1.0) +.. literalinclude:: ../../examples/tutorial/pure/A_equivalent.pyx Notice how in order to provide the Python wrappers to the definitions in the :file:`.pxd`, that is, to be accessible from Python, @@ -109,7 +82,7 @@ In the example above, the type of the local variable `a` in `myfunction()` -is not fixed and will thus be a Python object. To statically type it, one +is not fixed and will thus be a :term:`Python object`. To statically type it, one can use Cython's ``@cython.locals`` decorator (see :ref:`magic_attributes`, and :ref:`magic_attributes_pxd`). @@ -141,12 +114,7 @@ * ``compiled`` is a special variable which is set to ``True`` when the compiler runs, and ``False`` in the interpreter. Thus, the code - :: - - if cython.compiled: - print("Yep, I'm compiled.") - else: - print("Just a lowly interpreted script.") + .. literalinclude:: ../../examples/tutorial/pure/compiled_switch.py will behave differently depending on whether or not the code is executed as a compiled extension (:file:`.so`/:file:`.pyd`) module or a plain :file:`.py` @@ -159,58 +127,36 @@ * ``cython.declare`` declares a typed variable in the current scope, which can be used in place of the :samp:`cdef type var [= value]` construct. This has two forms, the first as an assignment (useful as it creates a declaration in interpreted - mode as well):: - - x = cython.declare(cython.int) # cdef int x - y = cython.declare(cython.double, 0.57721) # cdef double y = 0.57721 + mode as well): - and the second mode as a simple function call:: + .. literalinclude:: ../../examples/tutorial/pure/cython_declare.py - cython.declare(x=cython.int, y=cython.double) # cdef int x; cdef double y + and the second mode as a simple function call: - It can also be used to type class constructors:: + .. literalinclude:: ../../examples/tutorial/pure/cython_declare2.py - class A: - cython.declare(a=cython.int, b=cython.int) - def __init__(self, b=0): - self.a = 3 - self.b = b + It can also be used to define extension type private, readonly and public attributes: - And even to define extension type private, readonly and public attributes:: - - @cython.cclass - class A: - cython.declare(a=cython.int, b=cython.int) - c = cython.declare(cython.int, visibility='public') - d = cython.declare(cython.int, 5) # private by default. - e = cython.declare(cython.int, 5, visibility='readonly') + .. literalinclude:: ../../examples/tutorial/pure/cclass.py * ``@cython.locals`` is a decorator that is used to specify the types of local - variables in the function body (including the arguments):: + variables in the function body (including the arguments): - @cython.locals(a=cython.double, b=cython.double, n=cython.p_double) - def foo(a, b, x, y): - n = a*b - ... + .. literalinclude:: ../../examples/tutorial/pure/locals.py * ``@cython.returns()`` specifies the function's return type. -* Starting with Cython 0.21, Python signature annotations can be used to - declare argument types. Cython recognises three ways to do this, as - shown in the following example. Note that it currently needs to be - enabled explicitly with the directive ``annotation_typing=True``. - This might change in a later version. - - :: - - # cython: annotation_typing=True +* ``@cython.exceptval(value=None, *, check=False)`` specifies the function's exception + return value and exception check semantics as follows:: - def func(plain_python_type: dict, - named_python_type: 'dict', - explicit_python_type: {'type': dict}, - explicit_c_type: {'ctype': 'int'}): - ... + @exceptval(-1) # cdef int func() except -1: + @exceptval(-1, check=False) # cdef int func() except -1: + @exceptval(check=True) # cdef int func() except *: + @exceptval(-1, check=True) # cdef int func() except? -1: + @exceptval(check=False) # no exception checking/propagation + If exception propagation is disabled, any Python exceptions that are raised + inside of the function will be printed and ignored. C types ^^^^^^^ @@ -250,6 +196,10 @@ * ``@cython.inline`` is the equivalent of the C ``inline`` modifier. +* ``@cython.final`` terminates the inheritance chain by preventing a type from + being used as a base class, or a method from being overridden in subtypes. + This enables certain optimisations such as inlined method calls. + Here is an example of a :keyword:`cdef` function:: @cython.cfunc @@ -259,6 +209,63 @@ return a == b +Managing the Global Interpreter Lock +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``cython.nogil`` can be used as a context manager or as a decorator to replace the :keyword:`nogil` keyword:: + + with cython.nogil: + # code block with the GIL released + + @cython.nogil + @cython.cfunc + def func_released_gil() -> cython.int: + # function with the GIL released + +* ``cython.gil`` can be used as a context manager to replace the :keyword:`gil` keyword:: + + with cython.gil: + # code block with the GIL acquired + + .. Note:: Cython currently does not support the ``@cython.with_gil`` decorator. + +Both directives accept an optional boolean parameter for conditionally +releasing or acquiring the GIL. The condition must be constant (at compile time):: + + with cython.nogil(False): + # code block with the GIL not released + + @cython.nogil(True) + @cython.cfunc + def func_released_gil() -> cython.int: + # function with the GIL released + + with cython.gil(False): + # code block with the GIL not acquired + + with cython.gil(True): + # code block with the GIL acquired + +A common use case for conditionally acquiring and releasing the GIL are fused types +that allow different GIL handling depending on the specific type (see :ref:`gil_conditional`). + +cimports +^^^^^^^^ + +The special ``cython.cimports`` package name gives access to cimports +in code that uses Python syntax. Note that this does not mean that C +libraries become available to Python code. It only means that you can +tell Cython what cimports you want to use, without requiring special +syntax. Running such code in plain Python will fail. + +.. literalinclude:: ../../examples/tutorial/pure/py_cimport.py + +Since such code must necessarily refer to the non-existing +``cython.cimports`` 'package', the plain cimport form +``cimport cython.cimports...`` is not available. +You must use the form ``from cython.cimports...``. + + Further Cython functions and declarations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -273,8 +280,15 @@ :: cython.declare(n=cython.longlong) - print cython.sizeof(cython.longlong) - print cython.sizeof(n) + print(cython.sizeof(cython.longlong)) + print(cython.sizeof(n)) + +* ``typeof`` returns a string representation of the argument's type for debugging purposes. It can take expressions. + + :: + + cython.declare(n=cython.longlong) + print(cython.typeof(n)) * ``struct`` can be used to create struct types.:: @@ -312,53 +326,92 @@ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The special `cython` module can also be imported and used within the augmenting -:file:`.pxd` file. For example, the following Python file :file:`dostuff.py`:: - - def dostuff(n): - t = 0 - for i in range(n): - t += i - return t +:file:`.pxd` file. For example, the following Python file :file:`dostuff.py`: -can be augmented with the following :file:`.pxd` file :file:`dostuff.pxd`:: +.. literalinclude:: ../../examples/tutorial/pure/dostuff.py - import cython +can be augmented with the following :file:`.pxd` file :file:`dostuff.pxd`: - @cython.locals(t = cython.int, i = cython.int) - cpdef int dostuff(int n) +.. literalinclude:: ../../examples/tutorial/pure/dostuff.pxd The :func:`cython.declare()` function can be used to specify types for global variables in the augmenting :file:`.pxd` file. +.. _pep484_type_annotations: + +PEP-484 type annotations +------------------------ + +Python `type hints `_ +can be used to declare argument types, as shown in the +following example. To avoid conflicts with other kinds of annotation +usages, this can be disabled with the directive ``annotation_typing=False``. + + .. literalinclude:: ../../examples/tutorial/pure/annotations.py + +Note the use of ``cython.int`` rather than ``int`` - Cython does not translate +an ``int`` annotation to a C integer by default since the behaviour can be +quite different with respect to overflow and division. + +Annotations can be combined with the ``@cython.exceptval()`` decorator for non-Python +return types: + + .. literalinclude:: ../../examples/tutorial/pure/exceptval.py + +Note that the default exception handling behaviour when returning C numeric types +is to check for ``-1``, and if that was returned, check Python's error indicator +for an exception. This means, if no ``@exceptval`` decorator is provided, and the +return type is a numeric type, then the default with type annotations is +``@exceptval(-1, check=True)``, in order to make sure that exceptions are correctly +and efficiently reported to the caller. Exception propagation can be disabled +explicitly with ``@exceptval(check=False)``, in which case any Python exceptions +raised inside of the function will be printed and ignored. + +Since version 0.27, Cython also supports the variable annotations defined +in `PEP 526 `_. This allows to +declare types of variables in a Python 3.6 compatible way as follows: + +.. literalinclude:: ../../examples/tutorial/pure/pep_526.py + +There is currently no way to express the visibility of object attributes. + +``typing`` Module +^^^^^^^^^^^^^^^^^ + +Support for the full range of annotations described by PEP-484 is not yet +complete. Cython 3 currently understands the following features from the +``typing`` module: + +* ``Optional[tp]``, which is interpreted as ``tp or None``; +* typed containers such as ``List[str]``, which is interpreted as ``list``. The + hint that the elements are of type ``str`` is currently ignored; +* ``Tuple[...]``, which is converted into a Cython C-tuple where possible + and a regular Python ``tuple`` otherwise. +* ``ClassVar[...]``, which is understood in the context of + ``cdef class`` or ``@cython.cclass``. + +Some of the unsupported features are likely to remain +unsupported since these type hints are not relevant for the compilation to +efficient C code. In other cases, however, where the generated C code could +benefit from these type hints but does not currently, help is welcome to +improve the type analysis in Cython. Tips and Tricks --------------- +.. _calling-c-functions: + Calling C functions ^^^^^^^^^^^^^^^^^^^ Normally, it isn't possible to call C functions in pure Python mode as there is no general way to support it in normal (uncompiled) Python. However, in cases where an equivalent Python function exists, this can be achieved by -combining C function coercion with a conditional import as follows:: +combining C function coercion with a conditional import as follows: - # in mymodule.pxd: +.. literalinclude:: ../../examples/tutorial/pure/mymodule.pxd - # declare a C function as "cpdef" to export it to the module - cdef extern from "math.h": - cpdef double sin(double x) - - - # in mymodule.py: - - import cython - - # override with Python import if not in compiled code - if not cython.compiled: - from math import sin - - # calls sin() from math.h when compiled with Cython and math.sin() in Python - print(sin(0)) +.. literalinclude:: ../../examples/tutorial/pure/mymodule.py Note that the "sin" function will show up in the module namespace of "mymodule" here (i.e. there will be a ``mymodule.sin()`` function). You can mark it as an @@ -375,24 +428,11 @@ Using C arrays for fixed size lists ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Since Cython 0.22, C arrays can automatically coerce to Python lists or tuples. +C arrays can automatically coerce to Python lists or tuples. This can be exploited to replace fixed size Python lists in Python code by C -arrays when compiled. An example:: - - import cython +arrays when compiled. An example: - @cython.locals(counts=cython.int[10], digit=cython.int) - def count_digits(digits): - """ - >>> digits = '01112222333334445667788899' - >>> count_digits(map(int, digits)) - [1, 3, 4, 5, 3, 1, 2, 2, 3, 2] - """ - counts = [0] * 10 - for digit in digits: - assert 0 <= digit <= 9 - counts[digit] += 1 - return counts +.. literalinclude:: ../../examples/tutorial/pure/c_arrays.py In normal Python, this will use a Python list to collect the counts, whereas Cython will generate C code that uses a C array of C ints. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/pxd_files.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/pxd_files.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/pxd_files.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/pxd_files.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,3 +1,5 @@ +.. _pxd_files: + pxd files ========= @@ -9,27 +11,25 @@ ``pxd`` files have many use-cases: - 1. They can be used for sharing external C declarations. - 2. They can contain functions which are well suited for inlining by - the C compiler. Such functions should be marked ``inline``, example: - :: +1. They can be used for sharing external C declarations. +2. They can contain functions which are well suited for inlining by + the C compiler. Such functions should be marked ``inline``, example:: cdef inline int int_min(int a, int b): return b if b < a else a - 3. When accompanying an equally named ``pyx`` file, they +3. When accompanying an equally named ``pyx`` file, they provide a Cython interface to the Cython module so that other Cython modules can communicate with it using a more efficient protocol than the Python one. In our integration example, we might break it up into ``pxd`` files like this: - 1. Add a ``cmath.pxd`` function which defines the C functions available from +1. Add a ``cmath.pxd`` function which defines the C functions available from the C ``math.h`` header file, like ``sin``. Then one would simply do ``from cmath cimport sin`` in ``integrate.pyx``. - 2. Add a ``integrate.pxd`` so that other modules written in Cython - can define fast custom functions to integrate. - :: +2. Add a ``integrate.pxd`` so that other modules written in Cython + can define fast custom functions to integrate:: cdef class Function: cpdef evaluate(self, double x) @@ -39,3 +39,37 @@ Note that if you have a cdef class with attributes, the attributes must be declared in the class declaration ``pxd`` file (if you use one), not the ``pyx`` file. The compiler will tell you about this. + + +__init__.pxd +^^^^^^^^^^^^ + +Cython also supports ``__init__.pxd`` files for declarations in package's +namespaces, similar to ``__init__.py`` files in Python. + +Continuing the integration example, we could package the module as follows: + +1. Place the module files in a directory tree as one usually would for + Python: + + .. code-block:: text + + CyIntegration/ + ├── __init__.pyx + ├── __init__.pxd + ├── integrate.pyx + └── integrate.pxd + +2. In ``__init__.pxd``, use ``cimport`` for any declarations that one + would want to be available from the package's main namespace:: + + from CyIntegration cimport integrate + + Other modules would then be able to use ``cimport`` on the package in + order to recursively gain faster, Cython access to the entire package + and the data declared in its modules:: + + cimport CyIntegration + + cpdef do_integration(CyIntegration.integrate.Function f): + return CyIntegration.integrate.integrate(f, 0., 2., 1) Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/tutorial/python_division.png and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/tutorial/python_division.png differ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/queue_example/cqueue.pxd cython-0.20.1+1~202203241016-9537/docs/src/tutorial/queue_example/cqueue.pxd --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/queue_example/cqueue.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/queue_example/cqueue.pxd 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -cdef extern from "libcalg/queue.h": - ctypedef struct Queue: - pass - ctypedef void* QueueValue - - Queue* queue_new() - void queue_free(Queue* queue) - - int queue_push_head(Queue* queue, QueueValue data) - QueueValue queue_pop_head(Queue* queue) - QueueValue queue_peek_head(Queue* queue) - - int queue_push_tail(Queue* queue, QueueValue data) - QueueValue queue_pop_tail(Queue* queue) - QueueValue queue_peek_tail(Queue* queue) - - int queue_is_empty(Queue* queue) diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/queue_example/queue.pyx cython-0.20.1+1~202203241016-9537/docs/src/tutorial/queue_example/queue.pyx --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/queue_example/queue.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/queue_example/queue.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,96 +0,0 @@ -cimport cqueue - -cdef class Queue: - - cdef cqueue.Queue* _c_queue - - def __cinit__(self): - self._c_queue = cqueue.queue_new() - if self._c_queue is NULL: - raise MemoryError() - - def __dealloc__(self): - if self._c_queue is not NULL: - cqueue.queue_free(self._c_queue) - - cpdef int append(self, int value) except -1: - if not cqueue.queue_push_tail(self._c_queue, value): - raise MemoryError() - return 0 - - cdef int extend(self, int* values, Py_ssize_t count) except -1: - cdef Py_ssize_t i - for i in range(count): - if not cqueue.queue_push_tail(self._c_queue, values[i]): - raise MemoryError() - return 0 - - cpdef int peek(self) except? 0: - cdef int value = cqueue.queue_peek_head(self._c_queue) - if value == 0: - # this may mean that the queue is empty, or that it - # happens to contain a 0 value - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return value - - cpdef int pop(self) except? 0: - cdef int value = cqueue.queue_pop_head(self._c_queue) - if value == 0: - # this may mean that the queue is empty, or that it - # happens to contain a 0 value - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return value - - def __bool__(self): # same as __nonzero__ in Python 2.x - return not cqueue.queue_is_empty(self._c_queue) - -DEF repeat_count=10000 - -def test_cy(): - cdef int i - cdef Queue q = Queue() - for i in range(repeat_count): - q.append(i) - for i in range(repeat_count): - q.peek() - while q: - q.pop() - -def test_py(): - cdef int i - q = Queue() - for i in range(repeat_count): - q.append(i) - for i in range(repeat_count): - q.peek() - while q: - q.pop() - -from collections import deque - -def test_deque(): - cdef int i - q = deque() - for i in range(repeat_count): - q.appendleft(i) - for i in range(repeat_count): - q[-1] - while q: - q.pop() - -repeat = range(repeat_count) - -def test_py_exec(): - q = Queue() - d = dict(q=q, repeat=repeat) - - exec u"""\ -for i in repeat: - q.append(9) -for i in repeat: - q.peek() -while q: - q.pop() -""" in d diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/readings.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/readings.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/readings.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/readings.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,10 @@ Further reading =============== -The main documentation is located at http://docs.cython.org/. Some +The main documentation is located at https://docs.cython.org/. Some recent features might not have documentation written yet, in such cases some notes can usually be found in the form of a Cython -Enhancement Proposal (CEP) on http://wiki.cython.org/enhancements. +Enhancement Proposal (CEP) on https://github.com/cython/cython/wiki/enhancements. [Seljebotn09]_ contains more information about Cython and NumPy arrays. If you intend to use Cython code in a multi-threaded setting, @@ -16,11 +16,11 @@ Finally, don't hesitate to ask questions (or post reports on successes!) on the Cython users mailing list [UserList]_. The Cython developer mailing list, [DevList]_, is also open to everybody, but -focusses on core development issues. Feel free to use it to report a +focuses on core development issues. Feel free to use it to report a clear bug, to ask for guidance if you have time to spare to develop Cython, or if you have suggestions for future development. -.. [DevList] Cython developer mailing list: http://mail.python.org/mailman/listinfo/cython-devel +.. [DevList] Cython developer mailing list: https://mail.python.org/mailman/listinfo/cython-devel .. [Seljebotn09] D. S. Seljebotn, Fast numerical computations with Cython, Proceedings of the 8th Python in Science Conference, 2009. -.. [UserList] Cython users mailing list: http://groups.google.com/group/cython-users +.. [UserList] Cython users mailing list: https://groups.google.com/group/cython-users diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/related_work.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/related_work.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/related_work.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/related_work.rst 2022-03-24 10:16:46.000000000 +0000 @@ -39,12 +39,13 @@ it does not support natively, and supports very few of the standard Python modules. -.. [ctypes] http://docs.python.org/library/ctypes.html. +.. [ctypes] https://docs.python.org/library/ctypes.html. .. there's also the original ctypes home page: http://python.net/crew/theller/ctypes/ -.. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ +.. + [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [ShedSkin] M. Dufour, J. Coughlan, ShedSkin, - http://code.google.com/p/shedskin/ -.. [SWIG] David M. Beazley et al., - SWIG: An Easy to Use Tool for Integrating Scripting Languages with C and C++, + https://github.com/shedskin/shedskin +.. [SWIG] David M. Beazley et al., + SWIG: An Easy to Use Tool for Integrating Scripting Languages with C and C++, http://www.swig.org. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/tutorial/strings.rst cython-0.20.1+1~202203241016-9537/docs/src/tutorial/strings.rst --- cython-0.20.1+1~201611251650-6686/docs/src/tutorial/strings.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/tutorial/strings.rst 2022-03-24 10:16:46.000000000 +0000 @@ -107,12 +107,26 @@ Passing byte strings -------------------- +we have dummy C functions declared in +a file called :file:`c_func.pyx` that we are going to reuse throughout this tutorial: + +.. literalinclude:: ../../examples/tutorial/string/c_func.pyx + +We make a corresponding :file:`c_func.pxd` to be able to cimport those functions: + +.. literalinclude:: ../../examples/tutorial/string/c_func.pxd + It is very easy to pass byte strings between C code and Python. When receiving a byte string from a C library, you can let Cython convert it into a Python byte string by simply assigning it to a Python variable:: + from c_func cimport c_call_returning_a_c_string + cdef char* c_string = c_call_returning_a_c_string() + if c_string is NULL: + ... # handle error + cdef bytes py_string = c_string A type cast to :obj:`object` or :obj:`bytes` will do the same thing:: @@ -133,15 +147,9 @@ terminating null byte. In many cases, the user code will know the length already, e.g. because a C function returned it. In this case, it is much more efficient to tell Cython the exact number of bytes by -slicing the C string:: - - cdef char* c_string = NULL - cdef Py_ssize_t length = 0 - - # get pointer and length from a C function - get_a_c_string(&c_string, &length) +slicing the C string. Here is an example: - py_bytes_string = c_string[:length] +.. literalinclude:: ../../examples/tutorial/string/slicing_c_string.pyx Here, no additional byte counting is required and ``length`` bytes from the ``c_string`` will be copied into the Python bytes object, including @@ -152,20 +160,14 @@ Note that the creation of the Python bytes string can fail with an exception, e.g. due to insufficient memory. If you need to :c:func:`free()` the string after the conversion, you should wrap -the assignment in a try-finally construct:: +the assignment in a try-finally construct: - from libc.stdlib cimport free - cdef bytes py_string - cdef char* c_string = c_call_creating_a_new_c_string() - try: - py_string = c_string - finally: - free(c_string) +.. literalinclude:: ../../examples/tutorial/string/try_finally.pyx To convert the byte string back into a C :c:type:`char*`, use the opposite assignment:: - cdef char* other_c_string = py_string + cdef char* other_c_string = py_string # other_c_string is a 0-terminated string. This is a very fast operation after which ``other_c_string`` points to the byte string buffer of the Python string itself. It is tied to the @@ -203,13 +205,7 @@ Depending on how (and where) the data is being processed, it may be a good idea to instead receive a 1-dimensional memory view, e.g. -:: - - def process_byte_data(unsigned char[:] data): - length = data.shape[0] - first_byte = data[0] - slice_view = data[1:-1] - ... +.. literalinclude:: ../../examples/tutorial/string/arg_memview.pyx Cython's memory views are described in more detail in :doc:`../userguide/memoryviews`, but the above example already shows @@ -223,63 +219,34 @@ data, they would otherwise keep the entire original buffer alive. The general idea here is to be liberal with input by accepting any kind of byte buffer, but strict with output by returning a simple, well adapted -object. This can simply be done as follows:: +object. This can simply be done as follows: + +.. literalinclude:: ../../examples/tutorial/string/return_memview.pyx - def process_byte_data(unsigned char[:] data): - # ... process the data - if return_all: - return bytes(data) - else: - # example for returning a slice - return bytes(data[5:35]) - -If the byte input is actually encoded text, and the further processing -should happen at the Unicode level, then the right thing to do is to -decode the input straight away. This is almost only a problem in Python -2.x, where Python code expects that it can pass a byte string (:obj:`str`) -with encoded text into a text API. Since this usually happens in more -than one place in the module's API, a helper function is almost always the -way to go, since it allows for easy adaptation of the input normalisation -process later. +For read-only buffers, like :obj:`bytes`, the memoryview item type should +be declared as ``const`` (see :ref:`readonly_views`). If the byte input is +actually encoded text, and the further processing should happen at the +Unicode level, then the right thing to do is to decode the input straight +away. This is almost only a problem in Python 2.x, where Python code +expects that it can pass a byte string (:obj:`str`) with encoded text into +a text API. Since this usually happens in more than one place in the +module's API, a helper function is almost always the way to go, since it +allows for easy adaptation of the input normalisation process later. This kind of input normalisation function will commonly look similar to -the following:: +the following: - from cpython.version cimport PY_MAJOR_VERSION +.. literalinclude:: ../../examples/tutorial/string/to_unicode.pyx - cdef unicode _ustring(s): - if type(s) is unicode: - # fast path for most common case(s) - return s - elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes): - # only accept byte strings in Python 2.x, not in Py3 - return (s).decode('ascii') - elif isinstance(s, unicode): - # an evil cast to might work here in some(!) cases, - # depending on what the further processing does. to be safe, - # we can always create a copy instead - return unicode(s) - else: - raise TypeError(...) - -And should then be used like this:: - - def api_func(s): - text = _ustring(s) - ... +And should then be used like this: + +.. literalinclude:: ../../examples/tutorial/string/api_func.pyx Similarly, if the further processing happens at the byte level, but Unicode string input should be accepted, then the following might work, if you are -using memory views:: - - # define a global name for whatever char type is used in the module - ctypedef unsigned char char_type +using memory views: - cdef char_type[:] _chars(s): - if isinstance(s, unicode): - # encode to the specific encoding used inside of the module - s = (s).encode('utf8') - return s +.. literalinclude:: ../../examples/tutorial/string/to_char.pyx In this case, you might want to additionally ensure that byte string input really uses the correct encoding, e.g. if you require pure ASCII @@ -295,52 +262,13 @@ that they will not modify a string, or to require that users must not modify a string they return, for example: -.. code-block:: c +.. literalinclude:: ../../examples/tutorial/string/someheader.h - typedef const char specialChar; - int process_string(const char* s); - const unsigned char* look_up_cached_string(const unsigned char* key); - -Since version 0.18, Cython has support for the ``const`` modifier in +Cython has support for the ``const`` modifier in the language, so you can declare the above functions straight away as -follows:: +follows: - cdef extern from "someheader.h": - ctypedef const char specialChar - int process_string(const char* s) - const unsigned char* look_up_cached_string(const unsigned char* key) - -Previous versions required users to make the necessary declarations -at a textual level. If you need to support older Cython versions, -you can use the following approach. - -In general, for arguments of external C functions, the ``const`` -modifier does not matter and can be left out in the Cython -declaration (e.g. in a .pxd file). The C compiler will still do -the right thing, even if you declare this to Cython:: - - cdef extern from "someheader.h": - int process_string(char* s) # note: looses API information! - -However, in most other situations, such as for return values and -variables that use specifically typedef-ed API types, it does matter -and the C compiler will emit at least a warning if used incorrectly. -To help with this, you can use the type definitions in the -``libc.string`` module, e.g.:: - - from libc.string cimport const_char, const_uchar - - cdef extern from "someheader.h": - ctypedef const_char specialChar - int process_string(const_char* s) - const_uchar* look_up_cached_string(const_uchar* key) - -Note: even if the API only uses ``const`` for function arguments, -it is still preferable to properly declare them using these -provided :c:type:`const_char` types in order to simplify adaptations. -In Cython 0.18, these standard declarations have been changed to -use the correct ``const`` modifier, so your code will automatically -benefit from the new ``const`` support if it uses them. +.. literalinclude:: ../../examples/tutorial/string/const.pyx Decoding bytes to text @@ -358,20 +286,13 @@ ustring = byte_string.decode('UTF-8') Cython allows you to do the same for a C string, as long as it -contains no null bytes:: - - cdef char* some_c_string = c_call_returning_a_c_string() - ustring = some_c_string.decode('UTF-8') +contains no null bytes: -And, more efficiently, for strings where the length is known:: +.. literalinclude:: ../../examples/tutorial/string/naive_decode.pyx - cdef char* c_string = NULL - cdef Py_ssize_t length = 0 +And, more efficiently, for strings where the length is known: - # get pointer and length from a C function - get_a_c_string(&c_string, &length) - - ustring = c_string[:length].decode('UTF-8') +.. literalinclude:: ../../examples/tutorial/string/decode.pyx The same should be used when the string contains null bytes, e.g. when it uses an encoding like UCS-4, where each character is encoded in four @@ -379,7 +300,7 @@ Again, no bounds checking is done if slice indices are provided, so incorrect indices lead to data corruption and crashes. However, using -negative indices is possible since Cython 0.17 and will inject a call +negative indices is possible and will inject a call to :c:func:`strlen()` in order to determine the string length. Obviously, this only works for 0-terminated strings without internal null bytes. Text encoded in UTF-8 or one of the ISO-8859 encodings is @@ -389,23 +310,9 @@ It is common practice to wrap string conversions (and non-trivial type conversions in general) in dedicated functions, as this needs to be done in exactly the same way whenever receiving text from C. This -could look as follows:: - - from libc.stdlib cimport free +could look as follows: - cdef unicode tounicode(char* s): - return s.decode('UTF-8', 'strict') - - cdef unicode tounicode_with_length( - char* s, size_t length): - return s[:length].decode('UTF-8', 'strict') - - cdef unicode tounicode_with_length_and_free( - char* s, size_t length): - try: - return s[:length].decode('UTF-8', 'strict') - finally: - free(s) +.. literalinclude:: ../../examples/tutorial/string/utf_eight.pyx Most likely, you will prefer shorter function names in your code based on the kind of string being handled. Different types of content often @@ -442,18 +349,9 @@ When wrapping a C++ library, strings will usually come in the form of the :c:type:`std::string` class. As with C strings, Python byte strings -automatically coerce from and to C++ strings:: - - # distutils: language = c++ +automatically coerce from and to C++ strings: - from libcpp.string cimport string - - cdef string s = py_bytes_object - try: - s.append('abc') - py_bytes_object = s - finally: - del s +.. literalinclude:: ../../examples/tutorial/string/cpp_string.pyx The memory management situation is different than in C because the creation of a C++ string makes an independent copy of the string @@ -469,12 +367,9 @@ and then copies its buffer into a new C++ string. For the other direction, efficient decoding support is available -in Cython 0.17 and later:: - - cdef string s = string(b'abcdefg') +in Cython 0.17 and later: - ustring1 = s.decode('UTF-8') - ustring2 = s[2:-2].decode('UTF-8') +.. literalinclude:: ../../examples/tutorial/string/decode_cpp_string.pyx For C++ strings, decoding slices will always take the proper length of the string into account and apply Python slicing semantics (e.g. @@ -496,54 +391,30 @@ objects can reduce the code overhead a little. In this case, you can set the ``c_string_type`` directive in your module to :obj:`unicode` and the ``c_string_encoding`` to the encoding that your C code uses, -for example:: +for example: - # cython: c_string_type=unicode, c_string_encoding=utf8 - - cdef char* c_string = 'abcdefg' - - # implicit decoding: - cdef object py_unicode_object = c_string - - # explicit conversion to Python bytes: - py_bytes_object = c_string +.. literalinclude:: ../../examples/tutorial/string/auto_conversion_1.pyx The second use case is when all C strings that are being processed only contain ASCII encodable characters (e.g. numbers) and you want your code to use the native legacy string type in Python 2 for them, instead of always using Unicode. In this case, you can set the -string type to :obj:`str`:: - - # cython: c_string_type=str, c_string_encoding=ascii +string type to :obj:`str`: - cdef char* c_string = 'abcdefg' - - # implicit decoding in Py3, bytes conversion in Py2: - cdef object py_str_object = c_string - - # explicit conversion to Python bytes: - py_bytes_object = c_string - - # explicit conversion to Python unicode: - py_bytes_object = c_string +.. literalinclude:: ../../examples/tutorial/string/auto_conversion_2.pyx The other direction, i.e. automatic encoding to C strings, is only -supported for the ASCII codec (and the "default encoding", which is -runtime specific and may or may not be ASCII). This is because -CPython handles the memory management in this case by keeping an -encoded copy of the string alive together with the original unicode -string. Otherwise, there would be no way to limit the lifetime of -the encoded string in any sensible way, thus rendering any attempt to -extract a C string pointer from it a dangerous endeavour. As long -as you stick to the ASCII encoding for the ``c_string_encoding`` -directive, though, the following will work:: - - # cython: c_string_type=unicode, c_string_encoding=ascii - - def func(): - ustring = u'abc' - cdef char* s = ustring - return s[0] # returns u'a' +supported for ASCII and the "default encoding", which is usually UTF-8 +in Python 3 and usually ASCII in Python 2. CPython handles the memory +management in this case by keeping an encoded copy of the string alive +together with the original unicode string. Otherwise, there would be no +way to limit the lifetime of the encoded string in any sensible way, +thus rendering any attempt to extract a C string pointer from it a +dangerous endeavour. The following safely converts a Unicode string to +ASCII (change ``c_string_encoding`` to ``default`` to use the default +encoding instead): + +.. literalinclude:: ../../examples/tutorial/string/auto_conversion_3.pyx (This example uses a function context in order to safely control the lifetime of the Unicode string. Global Python variables can be @@ -573,7 +444,7 @@ encodes efficiently. This makes it a very good choice for source code files which usually consist mostly of ASCII characters. -.. _`UTF-8`: http://en.wikipedia.org/wiki/UTF-8 +.. _`UTF-8`: https://en.wikipedia.org/wiki/UTF-8 As an example, putting the following line into a UTF-8 encoded source file will print ``5``, as UTF-8 encodes the letter ``'ö'`` in the two @@ -600,7 +471,7 @@ the parser to read all unprefixed :obj:`str` literals in a source file as unicode string literals, just like Python 3. -.. _`CEP 108`: http://wiki.cython.org/enhancements/stringliterals +.. _`CEP 108`: https://github.com/cython/cython/wiki/enhancements-stringliterals Single bytes and characters --------------------------- @@ -608,7 +479,7 @@ The Python C-API uses the normal C :c:type:`char` type to represent a byte value, but it has two special integer types for a Unicode code point value, i.e. a single Unicode character: :c:type:`Py_UNICODE` -and :c:type:`Py_UCS4`. Since version 0.13, Cython supports the +and :c:type:`Py_UCS4`. Cython supports the first natively, support for :c:type:`Py_UCS4` is new in Cython 0.15. :c:type:`Py_UNICODE` is either defined as an unsigned 2-byte or 4-byte integer, or as :c:type:`wchar_t`, depending on the platform. @@ -686,7 +557,7 @@ For more information on this topic, it is worth reading the `Wikipedia article about the UTF-16 encoding`_. -.. _`Wikipedia article about the UTF-16 encoding`: http://en.wikipedia.org/wiki/UTF-16/UCS-2 +.. _`Wikipedia article about the UTF-16 encoding`: https://en.wikipedia.org/wiki/UTF-16/UCS-2 The same properties apply to Cython code that gets compiled for a narrow CPython runtime environment. In most cases, e.g. when @@ -745,30 +616,18 @@ Cython 0.13 supports efficient iteration over :c:type:`char*`, bytes and unicode strings, as long as the loop variable is appropriately typed. So the following will generate the expected -C code:: +C code: - cdef char* c_string = ... +.. literalinclude:: ../../examples/tutorial/string/for_char.pyx - cdef char c - for c in c_string[:100]: - if c == 'A': ... +The same applies to bytes objects: -The same applies to bytes objects:: - - cdef bytes bytes_string = ... - - cdef char c - for c in bytes_string: - if c == 'A': ... +.. literalinclude:: ../../examples/tutorial/string/for_bytes.pyx For unicode objects, Cython will automatically infer the type of the -loop variable as :c:type:`Py_UCS4`:: - - cdef unicode ustring = ... +loop variable as :c:type:`Py_UCS4`: - # NOTE: no typing required for 'uchar' ! - for uchar in ustring: - if uchar == u'A': ... +.. literalinclude:: ../../examples/tutorial/string/for_unicode.pyx The automatic type inference usually leads to much more efficient code here. However, note that some unicode operations still require the @@ -781,11 +640,9 @@ it. There are also optimisations for ``in`` tests, so that the following -code will run in plain C code, (actually using a switch statement):: +code will run in plain C code, (actually using a switch statement): - cdef Py_UCS4 uchar_val = get_a_unicode_character() - if uchar_val in u'abcABCxY': - ... +.. literalinclude:: ../../examples/tutorial/string/if_char_in.pyx Combined with the looping optimisation above, this can result in very efficient character switching code, e.g. in unicode parsers. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/two-syntax-variants-used cython-0.20.1+1~202203241016-9537/docs/src/two-syntax-variants-used --- cython-0.20.1+1~201611251650-6686/docs/src/two-syntax-variants-used 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/two-syntax-variants-used 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,18 @@ +.. note:: + + This page uses two different syntax variants: + + * Cython specific ``cdef`` syntax, which was designed to make type declarations + concise and easily readable from a C/C++ perspective. + + * Pure Python syntax which allows static Cython type declarations in + :ref:`pure Python code `, + following `PEP-484 `_ type hints + and `PEP 526 `_ variable annotations. + + To make use of C data types in Python syntax, you need to import the special + ``cython`` module in the Python module that you want to compile, e.g. + + .. code-block:: python + + import cython diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/buffer.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/buffer.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/buffer.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/buffer.rst 2022-03-24 10:16:46.000000000 +0000 @@ -16,21 +16,7 @@ where the number of columns is fixed at construction time but rows can be added dynamically. -:: - - # matrix.pyx - from libcpp.vector cimport vector - - cdef class Matrix: - cdef unsigned ncols - cdef vector[float] v - - def __cinit__(self, unsigned ncols): - self.ncols = ncols - - def add_row(self): - """Adds a row, initially zero-filled.""" - self.v.extend(self.ncols) +.. literalinclude:: ../../examples/userguide/buffer/matrix.pyx There are no methods to do anything productive with the matrices' contents. We could implement custom ``__getitem__``, ``__setitem__``, etc. for this, @@ -41,51 +27,7 @@ ``__getbuffer__`` and ``__releasebuffer__``, which Cython handles specially. -:: - - from cpython cimport Py_buffer - from libcpp.vector cimport vector - - cdef class Matrix: - cdef Py_ssize_t ncols - cdef Py_ssize_t shape[2] - cdef Py_ssize_t strides[2] - cdef vector[float] v - - def __cinit__(self, Py_ssize_t ncols): - self.ncols = ncols - - def add_row(self): - """Adds a row, initially zero-filled.""" - self.v.extend(self.ncols) - - def __getbuffer__(self, Py_buffer *buffer, int flags): - cdef Py_ssize_t itemsize = sizeof(self.v[0]) - - self.shape[0] = self.v.size() / self.ncols - self.shape[1] = self.ncols - - # Stride 1 is the distance, in bytes, between two items in a row; - # this is the distance between two adjacent items in the vector. - # Stride 0 is the distance between the first elements of adjacent rows. - self.strides[1] = ( &(self.v[1]) - - &(self.v[0])) - self.strides[0] = self.ncols * self.strides[1] - - buffer.buf = &(self.v[0]) - buffer.format = 'f' # float - buffer.internal = NULL # see References - buffer.itemsize = itemsize - buffer.len = self.v.size() * itemsize # product(shape) * itemsize - buffer.ndim = 2 - buffer.obj = self - buffer.readonly = 0 - buffer.shape = self.shape - buffer.strides = self.strides - buffer.suboffsets = NULL # for pointer arrays only - - def __releasebuffer__(self, Py_buffer *buffer): - pass +.. literalinclude:: ../../examples/userguide/buffer/matrix_with_buffer.pyx The method ``Matrix.__getbuffer__`` fills a descriptor structure, called a ``Py_buffer``, that is defined by the Python C-API. @@ -133,29 +75,7 @@ We can add a reference count to each matrix, and lock it for mutation whenever a view exists. -:: - - cdef class Matrix: - # ... - cdef int view_count - - def __cinit__(self, Py_ssize_t ncols): - self.ncols = ncols - self.view_count = 0 - - def add_row(self): - if self.view_count > 0: - raise ValueError("can't add row while being viewed") - self.v.resize(self.v.size() + self.ncols) - - def __getbuffer__(self, Py_buffer *buffer, int flags): - # ... as before - - self.view_count += 1 - - def __releasebuffer__(self, Py_buffer *buffer): - self.view_count -= 1 - +.. literalinclude:: ../../examples/userguide/buffer/view_count.pyx Flags ----- Binary files /tmp/tmp2d4wmwme/ShV34fQhQG/cython-0.20.1+1~201611251650-6686/docs/src/userguide/compute_typed_html.jpg and /tmp/tmp2d4wmwme/GdWEr1gunb/cython-0.20.1+1~202203241016-9537/docs/src/userguide/compute_typed_html.jpg differ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/debugging.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/debugging.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/debugging.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/debugging.rst 2022-03-24 10:16:46.000000000 +0000 @@ -21,19 +21,20 @@ make sudo make install +Installing the Cython debugger can be quite tricky. `This installation script and example code `_ might be useful. + The debugger will need debug information that the Cython compiler can export. This can be achieved from within the setup script by passing ``gdb_debug=True`` to ``cythonize()``:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup extensions = [Extension('source', ['source.pyx'])] setup(..., ext_modules=cythonize(extensions, gdb_debug=True)) For development it's often helpful to pass the ``--inplace`` flag to -the ``setup.py`` script, which makes distutils build your project +the ``setup.py`` script, which makes setuptools build your project "in place", i.e., not in a separate `build` directory. When invoking Cython from the command line directly you can have it write @@ -58,9 +59,11 @@ with an interpreter that is compiled with debugging symbols (i.e. configured with ``--with-pydebug`` or compiled with the ``-g`` CFLAG). If your Python is installed and managed by your package manager you probably need to install debug -support separately, e.g. for ubuntu:: +support separately. If using NumPy then you also need to install numpy debugging, or you'll +see an `import error for multiarray `_. +E.G. for ubuntu:: - $ sudo apt-get install python-dbg + $ sudo apt-get install python-dbg python-numpy-dbg $ python-dbg setup.py build_ext --inplace Then you need to run your script with ``python-dbg`` also. Ensure that when diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/early_binding_for_speed.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/early_binding_for_speed.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/early_binding_for_speed.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/early_binding_for_speed.rst 2022-03-24 10:16:46.000000000 +0000 @@ -22,22 +22,7 @@ For example, consider the following (silly) code example: -.. sourcecode:: cython - - cdef class Rectangle: - cdef int x0, y0 - cdef int x1, y1 - def __init__(self, int x0, int y0, int x1, int y1): - self.x0 = x0; self.y0 = y0; self.x1 = x1; self.y1 = y1 - def area(self): - area = (self.x1 - self.x0) * (self.y1 - self.y0) - if area < 0: - area = -area - return area - - def rectArea(x0, y0, x1, y1): - rect = Rectangle(x0, y0, x1, y1) - return rect.area() +.. literalinclude:: ../../examples/userguide/early_binding_for_speed/rectangle.pyx In the :func:`rectArea` method, the call to :meth:`rect.area` and the :meth:`.area` method contain a lot of Python overhead. @@ -45,26 +30,7 @@ However, in Cython, it is possible to eliminate a lot of this overhead in cases where calls occur within Cython code. For example: -.. sourcecode:: cython - - cdef class Rectangle: - cdef int x0, y0 - cdef int x1, y1 - def __init__(self, int x0, int y0, int x1, int y1): - self.x0 = x0; self.y0 = y0; self.x1 = x1; self.y1 = y1 - cdef int _area(self): - cdef int area - area = (self.x1 - self.x0) * (self.y1 - self.y0) - if area < 0: - area = -area - return area - def area(self): - return self._area() - - def rectArea(x0, y0, x1, y1): - cdef Rectangle rect - rect = Rectangle(x0, y0, x1, y1) - return rect._area() +.. literalinclude:: ../../examples/userguide/early_binding_for_speed/rectangle_cdef.pyx Here, in the Rectangle extension class, we have defined two different area calculation methods, the efficient :meth:`_area` C method, and the @@ -73,36 +39,14 @@ by declaring the local variable ``rect`` which is explicitly given the type Rectangle. By using this declaration, instead of just dynamically assigning to ``rect``, we gain the ability to access the much more efficient C-callable -:meth:`_rect` method. +:meth:`_area` method. But Cython offers us more simplicity again, by allowing us to declare dual-access methods - methods that can be efficiently called at C level, but can also be accessed from pure Python code at the cost of the Python access overheads. Consider this code: -.. sourcecode:: cython - - cdef class Rectangle: - cdef int x0, y0 - cdef int x1, y1 - def __init__(self, int x0, int y0, int x1, int y1): - self.x0 = x0; self.y0 = y0; self.x1 = x1; self.y1 = y1 - cpdef int area(self): - cdef int area - area = (self.x1 - self.x0) * (self.y1 - self.y0) - if area < 0: - area = -area - return area - - def rectArea(x0, y0, x1, y1): - cdef Rectangle rect - rect = Rectangle(x0, y0, x1, y1) - return rect.area() - -.. note:: - - in earlier versions of Cython, the :keyword:`cpdef` keyword is - ``rdef`` - but has the same effect). +.. literalinclude:: ../../examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx Here, we just have a single area method, declared as :keyword:`cpdef` to make it efficiently callable as a C function, but still accessible from pure Python @@ -111,18 +55,18 @@ If within Cython code, we have a variable already 'early-bound' (ie, declared explicitly as type Rectangle, (or cast to type Rectangle), then invoking its area method will use the efficient C code path and skip the Python overhead. -But if in Pyrex or regular Python code we have a regular object variable +But if in Cython or regular Python code we have a regular object variable storing a Rectangle object, then invoking the area method will require: * an attribute lookup for the area method * packing a tuple for arguments and a dict for keywords (both empty in this case) -* using the Python API to call the method +* using the Python API to call the method and within the area method itself: * parsing the tuple and keywords * executing the calculation code -* converting the result to a python object and returning it +* converting the result to a python object and returning it So within Cython, it is possible to achieve massive optimisations by using strong typing in declaration and casting of variables. For tight loops diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/extension_types.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/extension_types.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/extension_types.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/extension_types.rst 2022-03-24 10:16:46.000000000 +0000 @@ -11,20 +11,10 @@ As well as creating normal user-defined classes with the Python class statement, Cython also lets you create new built-in Python types, known as -extension types. You define an extension type using the :keyword:`cdef` class -statement. Here's an example:: +:term:`extension types`. You define an extension type using the :keyword:`cdef` class +statement. Here's an example: - cdef class Shrubbery: - - cdef int width, height - - def __init__(self, w, h): - self.width = w - self.height = h - - def describe(self): - print "This shrubbery is", self.width, \ - "by", self.height, "cubits." +.. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx As you can see, a Cython extension type definition looks a lot like a Python class definition. Within it, you use the def statement to define methods that @@ -35,18 +25,20 @@ attributes. The attributes may be Python objects (either generic or of a particular extension type), or they may be of any C data type. So you can use extension types to wrap arbitrary C data structures and provide a Python-like -interface to them. +interface to them. .. _readonly: -Attributes -============ +Static Attributes +================= Attributes of an extension type are stored directly in the object's C struct. The set of attributes is fixed at compile time; you can't add attributes to an extension type instance at run time simply by assigning to them, as you could -with a Python class instance. (You can subclass the extension type in Python -and add attributes to instances of the subclass, however.) +with a Python class instance. However, you can explicitly enable support +for dynamically assigned attributes, or subclass the extension type with a normal +Python class, which then supports arbitrary attribute assignments. +See :ref:`dynamic_attributes`. There are two ways that attributes of an extension type can be accessed: by Python attribute lookup, or by direct access to the C struct from Cython code. @@ -56,11 +48,9 @@ By default, extension type attributes are only accessible by direct access, not Python access, which means that they are not accessible from Python code. To make them accessible from Python code, you need to declare them as -:keyword:`public` or :keyword:`readonly`. For example:: +:keyword:`public` or :keyword:`readonly`. For example: - cdef class Shrubbery: - cdef public int width, height - cdef readonly float depth +.. literalinclude:: ../../examples/userguide/extension_types/python_access.pyx makes the width and height attributes readable and writable from Python code, and the depth attribute readable but not writable. @@ -76,6 +66,24 @@ Python access, not direct access. All the attributes of an extension type are always readable and writable by C-level access. + +.. _dynamic_attributes: + +Dynamic Attributes +================== + +It is not possible to add attributes to an extension type at runtime by default. +You have two ways of avoiding this limitation, both add an overhead when +a method is called from Python code. Especially when calling ``cpdef`` methods. + +The first approach is to create a Python subclass.: + +.. literalinclude:: ../../examples/userguide/extension_types/extendable_animal.pyx + +Declaring a ``__dict__`` attribute is the second way of enabling dynamic attributes.: + +.. literalinclude:: ../../examples/userguide/extension_types/dict_animal.pyx + Type declarations =================== @@ -97,60 +105,59 @@ -- the code will compile, but an attribute error will be raised at run time. The solution is to declare ``sh`` as being of type :class:`Shrubbery`, as -follows:: +follows: - cdef widen_shrubbery(Shrubbery sh, extra_width): - sh.width = sh.width + extra_width +.. literalinclude:: ../../examples/userguide/extension_types/widen_shrubbery.pyx Now the Cython compiler knows that ``sh`` has a C attribute called :attr:`width` and will generate code to access it directly and efficiently. -The same consideration applies to local variables, for example,:: +The same consideration applies to local variables, for example: - cdef Shrubbery another_shrubbery(Shrubbery sh1): - cdef Shrubbery sh2 - sh2 = Shrubbery() - sh2.width = sh1.width - sh2.height = sh1.height - return sh2 +.. literalinclude:: ../../examples/userguide/extension_types/shrubbery_2.pyx + +.. note:: + + We here ``cimport`` the class :class:`Shrubbery`, and this is necessary + to declare the type at compile time. To be able to ``cimport`` an extension type, + we split the class definition into two parts, one in a definition file and + the other in the corresponding implementation file. You should read + :ref:`sharing_extension_types` to learn to do that. Type Testing and Casting ------------------------ -Suppose I have a method :meth:`quest` which returns an object of type :class:`Shrubbery`. +Suppose I have a method :meth:`quest` which returns an object of type :class:`Shrubbery`. To access it's width I could write:: cdef Shrubbery sh = quest() - print sh.width + print(sh.width) -which requires the use of a local variable and performs a type test on assignment. +which requires the use of a local variable and performs a type test on assignment. If you *know* the return value of :meth:`quest` will be of type :class:`Shrubbery` you can use a cast to write:: - print (quest()).width + print( (quest()).width ) -This may be dangerous if :meth:`quest()` is not actually a :class:`Shrubbery`, as it -will try to access width as a C struct member which may not exist. At the C level, -rather than raising an :class:`AttributeError`, either an nonsensical result will be +This may be dangerous if :meth:`quest()` is not actually a :class:`Shrubbery`, as it +will try to access width as a C struct member which may not exist. At the C level, +rather than raising an :class:`AttributeError`, either an nonsensical result will be returned (interpreting whatever data is at that address as an int) or a segfault may result from trying to access invalid memory. Instead, one can write:: - print (quest()).width - -which performs a type check (possibly raising a :class:`TypeError`) before making the -cast and allowing the code to proceed. + print( (quest()).width ) -To explicitly test the type of an object, use the :meth:`isinstance` method. By default, -in Python, the :meth:`isinstance` method checks the :class:`__class__` attribute of the -first argument to determine if it is of the required type. However, this is potentially -unsafe as the :class:`__class__` attribute can be spoofed or changed, but the C structure -of an extension type must be correct to access its :keyword:`cdef` attributes and call its :keyword:`cdef` methods. Cython detects if the second argument is a known extension -type and does a type check instead, analogous to Pyrex's :meth:`typecheck`. -The old behavior is always available by passing a tuple as the second parameter:: +which performs a type check (possibly raising a :class:`TypeError`) before making the +cast and allowing the code to proceed. - print isinstance(sh, Shrubbery) # Check the type of sh - print isinstance(sh, (Shrubbery,)) # Check sh.__class__ +To explicitly test the type of an object, use the :meth:`isinstance` builtin function. +For known builtin or extension types, Cython translates these into a +fast and safe type check that ignores changes to +the object's ``__class__`` attribute etc., so that after a successful +:meth:`isinstance` test, code can rely on the expected C structure of the +extension type and its :keyword:`cdef` attributes and methods. +.. _extension_types_and_none: Extension types and None ========================= @@ -207,7 +214,7 @@ be ``None``. * When comparing a value with ``None``, keep in mind that, if ``x`` is a Python object, ``x is None`` and ``x is not None`` are very efficient because they - translate directly to C pointer comparisons, whereas ``x == None`` and + translate directly to C pointer comparisons, whereas ``x == None`` and ``x != None``, or simply using ``x`` as a boolean value (as in ``if x: ...``) will invoke Python operations and therefore be much slower. @@ -220,6 +227,8 @@ read it carefully before attempting to use any special methods in your extension types. +.. _properties: + Properties ============ @@ -260,7 +269,7 @@ def __del__(self): # This is called when the property is deleted. - + The :meth:`__get__`, :meth:`__set__` and :meth:`__del__` methods are all optional; if they are omitted, an exception will be raised when the @@ -269,7 +278,7 @@ Here's a complete example. It defines a property which adds to a list each time it is written to, returns the list when it is read, and empties the list when it is deleted.:: - + # cheesy.pyx cdef class CheeseShop: @@ -283,7 +292,7 @@ return "We don't have: %s" % self.cheeses @cheese.setter - def cheese(self): + def cheese(self, value): self.cheeses.append(value) @cheese.deleter @@ -294,18 +303,18 @@ from cheesy import CheeseShop shop = CheeseShop() - print shop.cheese + print(shop.cheese) shop.cheese = "camembert" - print shop.cheese + print(shop.cheese) shop.cheese = "cheddar" - print shop.cheese + print(shop.cheese) del shop.cheese - print shop.cheese + print(shop.cheese) -.. sourcecode:: text +.. code-block:: text # Test output We don't have: [] @@ -313,10 +322,13 @@ We don't have: ['camembert', 'cheddar'] We don't have: [] +.. _subclassing: + Subclassing ============= -An extension type may inherit from a built-in type or another extension type:: +If an extension type inherits from other types, the first base class must be +a built-in type or another extension type:: cdef class Parrot: ... @@ -331,14 +343,16 @@ must either be declared as an extern extension type or imported using the :keyword:`cimport` statement. -An extension type can only have one base class (no multiple inheritance). +Multiple inheritance is supported, however the second and subsequent base +classes must be an ordinary Python class (not an extension type or a built-in +type). Cython extension types can also be subclassed in Python. A Python class can inherit from multiple extension types provided that the usual Python rules for multiple inheritance are followed (i.e. the C layouts of all the base classes must be compatible). -Since Cython 0.13.1, there is a way to prevent extension types from +There is a way to prevent extension types from being subtyped in Python. This is done via the ``final`` directive, usually set on an extension type using a decorator:: @@ -371,24 +385,24 @@ cdef class Parrot: cdef void describe(self): - print "This parrot is resting." + print("This parrot is resting.") cdef class Norwegian(Parrot): cdef void describe(self): Parrot.describe(self) - print "Lovely plumage!" + print("Lovely plumage!") cdef Parrot p1, p2 p1 = Parrot() p2 = Norwegian() - print "p1:" + print("p1:") p1.describe() - print "p2:" + print("p2:") p2.describe() -.. sourcecode:: text +.. code-block:: text # Output p1: @@ -409,16 +423,17 @@ cdef class OwnedPointer: cdef void* ptr - cdef __dealloc__(self): - if ptr != NULL: - free(ptr) + def __dealloc__(self): + if self.ptr is not NULL: + free(self.ptr) @staticmethod cdef create(void* ptr): p = OwnedPointer() p.ptr = ptr - return ptr + return p +.. _forward_declaring_extension_types: Forward-declaring extension types =================================== @@ -463,8 +478,9 @@ ``__init__()`` method (again, as known from Python). Thus, in the example above, the first instantiation will print ``eating!``, but the second will not. This is only one of the reasons why the ``__cinit__()`` method is -safer and preferable over the normal ``__init__()`` method for extension -types. +safer than the normal ``__init__()`` method for initialising extension types +and bringing them into a correct and safe state. +See section :ref:`_initialisation_methods` about the differences. The second performance improvement applies to types that are often created and deleted in a row, so that they can benefit from a freelist. Cython @@ -483,6 +499,107 @@ penguin = None penguin = Penguin('fish 2') # does not need to allocate memory! +.. _existing-pointers-instantiation: + +Instantiation from existing C/C++ pointers +=========================================== + +It is quite common to want to instantiate an extension class from an existing +(pointer to a) data structure, often as returned by external C/C++ functions. + +As extension classes can only accept Python objects as arguments in their +constructors, this necessitates the use of factory functions. For example, :: + + from libc.stdlib cimport malloc, free + + # Example C struct + ctypedef struct my_c_struct: + int a + int b + + + cdef class WrapperClass: + """A wrapper class for a C/C++ data structure""" + cdef my_c_struct *_ptr + cdef bint ptr_owner + + def __cinit__(self): + self.ptr_owner = False + + def __dealloc__(self): + # De-allocate if not null and flag is set + if self._ptr is not NULL and self.ptr_owner is True: + free(self._ptr) + self._ptr = NULL + + # Extension class properties + @property + def a(self): + return self._ptr.a if self._ptr is not NULL else None + + @property + def b(self): + return self._ptr.b if self._ptr is not NULL else None + + @staticmethod + cdef WrapperClass from_ptr(my_c_struct *_ptr, bint owner=False): + """Factory function to create WrapperClass objects from + given my_c_struct pointer. + + Setting ``owner`` flag to ``True`` causes + the extension type to ``free`` the structure pointed to by ``_ptr`` + when the wrapper object is deallocated.""" + # Call to __new__ bypasses __init__ constructor + cdef WrapperClass wrapper = WrapperClass.__new__(WrapperClass) + wrapper._ptr = _ptr + wrapper.ptr_owner = owner + return wrapper + + @staticmethod + cdef WrapperClass new_struct(): + """Factory function to create WrapperClass objects with + newly allocated my_c_struct""" + cdef my_c_struct *_ptr = malloc(sizeof(my_c_struct)) + if _ptr is NULL: + raise MemoryError + _ptr.a = 0 + _ptr.b = 0 + return WrapperClass.from_ptr(_ptr, owner=True) + + +To then create a ``WrapperClass`` object from an existing ``my_c_struct`` +pointer, ``WrapperClass.from_ptr(ptr)`` can be used in Cython code. To allocate +a new structure and wrap it at the same time, ``WrapperClass.new_struct`` can be +used instead. + +It is possible to create multiple Python objects all from the same pointer +which point to the same in-memory data, if that is wanted, though care must be +taken when de-allocating as can be seen above. +Additionally, the ``ptr_owner`` flag can be used to control which +``WrapperClass`` object owns the pointer and is responsible for de-allocation - +this is set to ``False`` by default in the example and can be enabled by calling +``from_ptr(ptr, owner=True)``. + +The GIL must *not* be released in ``__dealloc__`` either, or another lock used +if it is, in such cases or race conditions can occur with multiple +de-allocations. + +Being a part of the object constructor, the ``__cinit__`` method has a Python +signature, which makes it unable to accept a ``my_c_struct`` pointer as an +argument. + +Attempts to use pointers in a Python signature will result in errors like:: + + Cannot convert 'my_c_struct *' to Python object + +This is because Cython cannot automatically convert a pointer to a Python +object, unlike with native types like ``int``. + +Note that for native types, Cython will copy the value and create a new Python +object while in the above case, data is not copied and deallocating memory is +a responsibility of the extension class. + +.. _making_extension_types_weak_referenceable: Making extension types weak-referenceable ========================================== @@ -494,14 +611,101 @@ cdef class ExplodingAnimal: """This animal will self-destruct when it is no longer strongly referenced.""" - + cdef object __weakref__ -Controlling cyclic garbage collection in CPython -================================================ +Controlling deallocation and garbage collection in CPython +========================================================== + +.. NOTE:: -By default each extension type will support the cyclic garbage collector of + This section only applies to the usual CPython implementation + of Python. Other implementations like PyPy work differently. + +.. _dealloc_intro: + +Introduction +------------ + +First of all, it is good to understand that there are two ways to +trigger deallocation of Python objects in CPython: +CPython uses reference counting for all objects and any object with a +reference count of zero is immediately deallocated. This is the most +common way of deallocating an object. For example, consider :: + + >>> x = "foo" + >>> x = "bar" + +After executing the second line, the string ``"foo"`` is no longer referenced, +so it is deallocated. This is done using the ``tp_dealloc`` slot, which can be +customized in Cython by implementing ``__dealloc__``. + +The second mechanism is the cyclic garbage collector. +This is meant to resolve cyclic reference cycles such as :: + + >>> class Object: + ... pass + >>> def make_cycle(): + ... x = Object() + ... y = [x] + ... x.attr = y + +When calling ``make_cycle``, a reference cycle is created since ``x`` +references ``y`` and vice versa. Even though neither ``x`` or ``y`` +are accessible after ``make_cycle`` returns, both have a reference count +of 1, so they are not immediately deallocated. At regular times, the garbage +collector runs, which will notice the reference cycle +(using the ``tp_traverse`` slot) and break it. +Breaking a reference cycle means taking an object in the cycle +and removing all references from it to other Python objects (we call this +*clearing* an object). Clearing is almost the same as deallocating, except +that the actual object is not yet freed. For ``x`` in the example above, +the attributes of ``x`` would be removed from ``x``. + +Note that it suffices to clear just one object in the reference cycle, +since there is no longer a cycle after clearing one object. Once the cycle +is broken, the usual refcount-based deallocation will actually remove the +objects from memory. Clearing is implemented in the ``tp_clear`` slot. +As we just explained, it is sufficient that one object in the cycle +implements ``tp_clear``. + +.. _trashcan: + +Enabling the deallocation trashcan +---------------------------------- + +In CPython, it is possible to create deeply recursive objects. For example:: + + >>> L = None + >>> for i in range(2**20): + ... L = [L] + +Now imagine that we delete the final ``L``. Then ``L`` deallocates +``L[0]``, which deallocates ``L[0][0]`` and so on until we reach a +recursion depth of ``2**20``. This deallocation is done in C and such +a deep recursion will likely overflow the C call stack, crashing Python. + +CPython invented a mechanism for this called the *trashcan*. It limits the +recursion depth of deallocations by delaying some deallocations. + +By default, Cython extension types do not use the trashcan but it can be +enabled by setting the ``trashcan`` directive to ``True``. For example:: + + cimport cython + @cython.trashcan(True) + cdef class Object: + cdef dict __dict__ + +Trashcan usage is inherited by subclasses +(unless explicitly disabled by ``@cython.trashcan(False)``). +Some builtin types like ``list`` use the trashcan, so subclasses of it +use the trashcan by default. + +Disabling cycle breaking (``tp_clear``) +--------------------------------------- + +By default, each extension type will support the cyclic garbage collector of CPython. If any Python objects can be referenced, Cython will automatically generate the ``tp_traverse`` and ``tp_clear`` slots. This is usually what you want. @@ -509,13 +713,13 @@ There is at least one reason why this might not be what you want: If you need to cleanup some external resources in the ``__dealloc__`` special function and your object happened to be in a reference cycle, the garbage collector may -have triggered a call to ``tp_clear`` to drop references. This is the way that -reference cycles are broken so that the garbage can actually be reclaimed. +have triggered a call to ``tp_clear`` to clear the object +(see :ref:`dealloc_intro`). -In that case any object references have vanished by the time when -``__dealloc__`` is called. Now your cleanup code lost access to the objects it -has to clean up. In that case you can disable the cycle breaker ``tp_clear`` -by using the ``no_gc_clear`` decorator :: +In that case, any object references have vanished when ``__dealloc__`` +is called. Now your cleanup code lost access to the objects it has to clean up. +To fix this, you can disable clearing instances of a specific class by using +the ``no_gc_clear`` directive:: @cython.no_gc_clear cdef class DBCursor: @@ -528,17 +732,21 @@ This example tries to close a cursor via a database connection when the Python object is destroyed. The ``DBConnection`` object is kept alive by the reference from ``DBCursor``. But if a cursor happens to be in a reference cycle, the -garbage collector may effectively "steal" the database connection reference, +garbage collector may delete the database connection reference, which makes it impossible to clean up the cursor. -Using the ``no_gc_clear`` decorator this can not happen anymore because the -references of a cursor object will not be cleared anymore. +If you use ``no_gc_clear``, it is important that any given reference cycle +contains at least one object *without* ``no_gc_clear``. Otherwise, the cycle +cannot be broken, which is a memory leak. + +Disabling cyclic garbage collection +----------------------------------- In rare cases, extension types can be guaranteed not to participate in cycles, but the compiler won't be able to prove this. This would be the case if the class can never reference itself, even indirectly. In that case, you can manually disable cycle collection by using the -``no_gc`` decorator, but beware that doing so when in fact the extension type +``no_gc`` directive, but beware that doing so when in fact the extension type can participate in cycles could cause memory leaks :: @cython.no_gc @@ -546,10 +754,29 @@ cdef str name cdef tuple addresses -If you can be sure addresses will contain only references to strings, +If you can be sure addresses will contain only references to strings, the above would be safe, and it may yield a significant speedup, depending on your usage pattern. + +.. _auto_pickle: + +Controlling pickling +==================== + +By default, Cython will generate a ``__reduce__()`` method to allow pickling +an extension type if and only if each of its members are convertible to Python +and it has no ``__cinit__`` method. +To require this behavior (i.e. throw an error at compile time if a class +cannot be pickled) decorate the class with ``@cython.auto_pickle(True)``. +One can also annotate with ``@cython.auto_pickle(False)`` to get the old +behavior of not generating a ``__reduce__`` method in any case. + +Manually implementing a ``__reduce__`` or `__reduce_ex__`` method will also +disable this auto-generation and can be used to support pickling of more +complicated types. + + Public and external extension types ==================================== @@ -558,6 +785,8 @@ Cython module. A public extension type declaration makes an extension type defined in a Cython module available to external C code. +.. _external_extension_types: + External extension types ------------------------ @@ -574,6 +803,8 @@ Here is an example which will let you get at the C-level members of the built-in complex object.:: + from __future__ import print_function + cdef extern from "complexobject.h": struct Py_complex: @@ -585,8 +816,8 @@ # A function which uses the above type def spam(complex c): - print "Real:", c.cval.real - print "Imag:", c.cval.imag + print("Real:", c.cval.real) + print("Imag:", c.cval.imag) .. note:: @@ -596,15 +827,23 @@ because, in the Python header files, the ``PyComplexObject`` struct is declared with: - .. sourcecode:: c + .. code-block:: c typedef struct { ... } PyComplexObject; + At runtime, a check will be performed when importing the Cython + c-extension module that ``__builtin__.complex``'s ``tp_basicsize`` + matches ``sizeof(`PyComplexObject)``. This check can fail if the Cython + c-extension module was compiled with one version of the + ``complexobject.h`` header but imported into a Python with a changed + header. This check can be tweaked by using ``check_size`` in the name + specification clause. + 2. As well as the name of the extension type, the module in which its type object can be found is also specified. See the implicit importing section - below. + below. 3. When declaring an external extension type, you don't declare any methods. Declaration of methods is not required in order to call them, @@ -613,6 +852,8 @@ declaration is inside a :keyword:`cdef` extern from block, you only need to declare those C members which you wish to access. +.. _name_specification_clause: + Name specification clause ------------------------- @@ -620,11 +861,24 @@ available for extern or public extension types. The full form of this clause is:: - [object object_struct_name, type type_object_name ] + [object object_struct_name, type type_object_name, check_size cs_option] + +Where: -where ``object_struct_name`` is the name to assume for the type's C struct, -and type_object_name is the name to assume for the type's statically declared -type object. (The object and type clauses can be written in either order.) +- ``object_struct_name`` is the name to assume for the type's C struct. +- ``type_object_name`` is the name to assume for the type's statically + declared type object. +- ``cs_option`` is ``warn`` (the default), ``error``, or ``ignore`` and is only + used for external extension types. If ``error``, the ``sizeof(object_struct)`` + that was found at compile time must match the type's runtime ``tp_basicsize`` + exactly, otherwise the module import will fail with an error. If ``warn`` + or ``ignore``, the ``object_struct`` is allowed to be smaller than the type's + ``tp_basicsize``, which indicates the runtime type may be part of an updated + module, and that the external module's developers extended the object in a + backward-compatible fashion (only adding new fields to the end of the object). + If ``warn``, a warning will be emitted in this case. + +The clauses can be written in any order. If the extension type declaration is inside a :keyword:`cdef` extern from block, the object clause is required, because Cython must be able to generate @@ -635,6 +889,96 @@ because Cython must be able to generate code that is compatible with external C code. +Attribute name matching and aliasing +------------------------------------ + +Sometimes the type's C struct as specified in ``object_struct_name`` may use +different labels for the fields than those in the ``PyTypeObject``. This can +easily happen in hand-coded C extensions where the ``PyTypeObject_Foo`` has a +getter method, but the name does not match the name in the ``PyFooObject``. In +NumPy, for instance, python-level ``dtype.itemsize`` is a getter for the C +struct field ``elsize``. Cython supports aliasing field names so that one can +write ``dtype.itemsize`` in Cython code which will be compiled into direct +access of the C struct field, without going through a C-API equivalent of +``dtype.__getattr__('itemsize')``. + +For example, we may have an extension module ``foo_extension``:: + + cdef class Foo: + cdef public int field0, field1, field2; + + def __init__(self, f0, f1, f2): + self.field0 = f0 + self.field1 = f1 + self.field2 = f2 + +but a C struct in a file ``foo_nominal.h``: + +.. code-block:: c + + typedef struct { + PyObject_HEAD + int f0; + int f1; + int f2; + } FooStructNominal; + +Note that the struct uses ``f0``, ``f1``, ``f2`` but they are ``field0``, +``field1``, and ``field2`` in ``Foo``. We are given this situation, including +a header file with that struct, and we wish to write a function to sum the +values. If we write an extension module ``wrapper``:: + + cdef extern from "foo_nominal.h": + + ctypedef class foo_extension.Foo [object FooStructNominal]: + cdef: + int field0 + int field1 + int feild2 + + def sum(Foo f): + return f.field0 + f.field1 + f.field2 + +then ``wrapper.sum(f)`` (where ``f = foo_extension.Foo(1, 2, 3)``) will still +use the C-API equivalent of:: + + return f.__getattr__('field0') + + f.__getattr__('field1') + + f.__getattr__('field1') + +instead of the desired C equivalent of ``return f->f0 + f->f1 + f->f2``. We can +alias the fields by using:: + + cdef extern from "foo_nominal.h": + + ctypedef class foo_extension.Foo [object FooStructNominal]: + cdef: + int field0 "f0" + int field1 "f1" + int field2 "f2" + + def sum(Foo f) except -1: + return f.field0 + f.field1 + f.field2 + +and now Cython will replace the slow ``__getattr__`` with direct C access to +the FooStructNominal fields. This is useful when directly processing Python +code. No changes to Python need be made to achieve significant speedups, even +though the field names in Python and C are different. Of course, one should +make sure the fields are equivalent. + +C inline properties +------------------- + +Similar to Python property attributes, Cython provides a way to declare C-level +properties on external extension types. This is often used to shadow Python +attributes through faster C level data access, but can also be used to add certain +functionality to existing types when using them from Cython. The declarations +must use `cdef inline`. + +For example, the above ``complex`` type could also be declared like this: + +.. literalinclude:: ../../examples/userguide/extension_types/c_property.pyx + Implicit importing ------------------ @@ -662,12 +1006,14 @@ an as clause, for example,:: cdef extern class My.Nested.Package.Spam as Yummy: - ... + ... which corresponds to the implicit import statement:: from My.Nested.Package import Spam as Yummy +.. _types_names_vs_constructor_names: + Type names vs. constructor names -------------------------------- @@ -698,6 +1044,8 @@ there are other ways that you could get hold of the constructor, but only Yummy is usable as a type name. +.. _public: + Public extension types ====================== @@ -706,5 +1054,29 @@ including the ``.h`` file in external C code that you write, that code can access the attributes of the extension type. +Dataclass extension types +========================= - +Cython supports extension types that behave like the dataclasses defined in +the Python 3.7+ standard library. The main benefit of using a dataclass is +that it can auto-generate simple ``__init__``, ``__repr__`` and comparison +functions. The Cython implementation behaves as much like the Python +standard library implementation as possible and therefore the documentation +here only briefly outlines the differences - if you plan on using them +then please read `the documentation for the standard library module +`_. + +Dataclasses can be declared using the ``@cython.dataclasses.dataclass`` +decorator on a Cython extension type. ``@cython.dataclasses.dataclass`` +can only be applied to extension types (types marked ``cdef`` or created with the +``cython.cclass`` decorator) and not to regular classes. If +you need to define special properties on a field then use ``cython.dataclasses.field`` + +.. literalinclude:: ../../examples/userguide/extension_types/dataclass.pyx + +You may use C-level types such as structs, pointers, or C++ classes. +However, you may find these types are not compatible with the auto-generated +special methods - for example if they cannot be converted from a Python +type they cannot be passed to a constructor, and so you must use a +``default_factory`` to initialize them. Like with the Python implementation, you can also control +which special functions an attribute is used in using ``field()``. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/external_C_code.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/external_C_code.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/external_C_code.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/external_C_code.rst 2022-03-24 10:16:46.000000000 +0000 @@ -17,7 +17,7 @@ Cython module can be used as a bridge to allow Python code to call C code, it can also be used to allow C code to call Python code. -.. _embedding Python: http://www.freenet.org.nz/python/embeddingpyrex/ +.. _embedding Python: https://web.archive.org/web/20120225082358/http://www.freenet.org.nz:80/python/embeddingpyrex/ External declarations ======================= @@ -52,7 +52,7 @@ The ``cdef extern`` from clause does three things: 1. It directs Cython to place a ``#include`` statement for the named header file in - the generated C code. + the generated C code. 2. It prevents Cython from generating any C code for the declarations found in the associated block. 3. It treats all declarations within the block as though they started with @@ -92,8 +92,8 @@ ctypedef int word will work okay whatever the actual size of a :c:type:`word` is (provided the header - file defines it correctly). Conversion to and from Python types, if any, will also - be used for this new type. + file defines it correctly). Conversion to and from Python types, if any, will also + be used for this new type. #. If the header file uses macros to define constants, translate them into a normal external variable declaration. You can also declare them as an @@ -118,7 +118,7 @@ pass * If you want to include a system header, put angle brackets inside the quotes:: - + cdef extern from "": ... @@ -129,6 +129,13 @@ cdef extern from *: ... +* If a ``cdef extern from "inc.h"`` block is not empty and contains only + function or variable declarations (and no type declarations of any kind), + Cython will put the ``#include "inc.h"`` statement after all + declarations generated by Cython. This means that the included file + has access to the variables, functions, structures, ... which are + declared by Cython. + Implementing functions in C --------------------------- @@ -182,19 +189,19 @@ +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ | C code | Possibilities for corresponding Cython Code | Comments | +=========================+=============================================+=======================================================================+ -| .. sourcecode:: c | :: | Cython will refer to the as ``struct Foo`` in the generated C code. | -| | | | +| .. code-block:: c | :: | Cython will refer to the type as ``struct Foo`` in | +| | | the generated C code. | | struct Foo { | cdef struct Foo: | | | ... | ... | | | }; | | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | Cython will refer to the type simply as ``Foo`` in | +| .. code-block:: c | :: | Cython will refer to the type simply as ``Foo`` in | | | | the generated C code. | | typedef struct { | ctypedef struct Foo: | | | ... | ... | | | } Foo; | | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | If the C header uses both a tag and a typedef with *different* | +| .. code-block:: c | :: | If the C header uses both a tag and a typedef with *different* | | | | names, you can use either form of declaration in Cython | | typedef struct foo { | cdef struct foo: | (although if you need to forward reference the type, | | ... | ... | you'll have to use the first form). | @@ -205,16 +212,49 @@ | | ctypedef struct Foo: | | | | ... | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | If the header uses the *same* name for the tag and typedef, you | +| .. code-block:: c | :: | If the header uses the *same* name for the tag and typedef, you | | | | won't be able to include a :keyword:`ctypedef` for it -- but then, | | typedef struct Foo { | cdef struct Foo: | it's not necessary. | | ... | ... | | | } Foo; | | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ +See also use of :ref:`external_extension_types`. Note that in all the cases below, you refer to the type in Cython code simply as :c:type:`Foo`, not ``struct Foo``. +Pointers +-------- +When interacting with a C-api there may be functions that require pointers as arguments. +Pointers are variables that contain a memory address to another variable. + +For example:: + + cdef extern from "": + cdef void increase_by_one(int *my_var) + +This function takes a pointer to an integer as argument. Knowing the address of the +integer allows the function to modify the value in place, so that the caller can see +the changes afterwards. In order to get the address from an existing variable, +use the ``&`` operator:: + + cdef int some_int = 42 + cdef int *some_int_pointer = &some_int + increase_by_one(some_int_pointer) + # Or without creating the extra variable + increase_by_one(&some_int) + print(some_int) # prints 44 (== 42+1+1) + +If you want to manipulate the variable the pointer points to, you can access it by +referencing its first element like you would in python ``my_pointer[0]``. For example:: + + cdef void increase_by_one(int *my_var): + my_var[0] += 1 + +For a deeper introduction to pointers, you can read `this tutorial at tutorialspoint +`_. For differences between +Cython and C syntax for manipulating pointers, see :ref:`statements_and_expressions`. + Accessing Python/C API routines --------------------------------- @@ -227,6 +267,10 @@ will allow you to create Python strings containing null bytes. +Note that Cython comes with ready-to-use declarations of (almost) all C-API functions +in the cimportable ``cpython.*`` modules. See the list in +https://github.com/cython/cython/tree/master/Cython/Includes/cpython + Special Types -------------- @@ -322,6 +366,50 @@ an advanced feature, only for the rare cases where everything else fails. +.. _verbatim_c: + +Including verbatim C code +------------------------- + +For advanced use cases, Cython allows you to directly write C code +as "docstring" of a ``cdef extern from`` block: + +.. literalinclude:: ../../examples/userguide/external_C_code/verbatim_c_code.pyx + +The above is essentially equivalent to having the C code in a file +``header.h`` and writing :: + + cdef extern from "header.h": + long square(long x) + void assign(long& x, long y) + +This feature is commonly used for platform specific adaptations at +compile time, for example: + +.. literalinclude:: ../../examples/userguide/external_C_code/platform_adaptation.pyx + +It is also possible to combine a header file and verbatim C code:: + + cdef extern from "badheader.h": + """ + /* This macro breaks stuff */ + #undef int + """ + # Stuff from badheader.h + +In this case, the C code ``#undef int`` is put right after +``#include "badheader.h"`` in the C code generated by Cython. + +Verbatim C code can also be used for version specific adaptations, e.g. when +a struct field was added to a library but is not available in older versions: + +.. literalinclude:: ../../examples/userguide/external_C_code/struct_field_adaptation.pyx + +Note that the string is parsed like any other docstring in Python. +If you require character escapes to be passed into the C code file, +use a raw docstring, i.e. ``r""" ... """``. + + Using Cython Declarations from C ================================ @@ -339,42 +427,67 @@ --------------------- You can make C types, variables and functions defined in a Cython module -accessible to C code that is linked with the module, by declaring them with -the public keyword:: +accessible to C code that is linked together with the Cython-generated C file, +by declaring them with the public keyword:: - cdef public struct Bunny: # public type declaration + cdef public struct Bunny: # a public type declaration int vorpalness - cdef public int spam # public variable declaration + cdef public int spam # a public variable declaration - cdef public void grail(Bunny *): # public function declaration - print "Ready the holy hand grenade" + cdef public void grail(Bunny *) # a public function declaration If there are any public declarations in a Cython module, a header file called :file:`modulename.h` file is generated containing equivalent C declarations for inclusion in other C code. -Users who are embedding Python in C with Cython need to make sure to call Py_Initialize() -and Py_Finalize(). For example, in the following snippet that includes :file:`modulename.h`:: +A typical use case for this is building an extension module from multiple +C sources, one of them being Cython generated (i.e. with something like +``Extension("grail", sources=["grail.pyx", "grail_helper.c"])`` in ``setup.py``. +In this case, the file ``grail_helper.c`` just needs to add +``#include "grail.h"`` in order to access the public Cython variables. + +A more advanced use case is embedding Python in C using Cython. +In this case, make sure to call Py_Initialize() and Py_Finalize(). +For example, in the following snippet that includes :file:`grail.h`: + +.. code-block:: c #include - #include "modulename.h" + #include "grail.h" - void grail() { + int main() { Py_Initialize(); - initmodulename(); + initgrail(); /* Python 2.x only ! */ Bunny b; grail(b); Py_Finalize(); } -Any C code wanting to make use of these declarations will need to be linked, -either statically or dynamically, with the extension module. +This C code can then be built together with the Cython-generated C code +in a single program (or library). + +In Python 3.x, calling the module init function directly should be avoided. Instead, +use the `inittab mechanism `_ +to link Cython modules into a single shared library or program. + +.. code-block:: c + + err = PyImport_AppendInittab("grail", PyInit_grail); + Py_Initialize(); + grail_module = PyImport_ImportModule("grail"); If the Cython module resides within a package, then the name of the ``.h`` file consists of the full dotted name of the module, e.g. a module called :mod:`foo.spam` would have a header file called :file:`foo.spam.h`. +.. NOTE:: + + On some operating systems like Linux, it is also possible to first + build the Cython extension in the usual way and then link against + the resulting ``.so`` file like a dynamic library. + Beware that this is not portable, so it should be avoided. + .. _api: C API Declarations @@ -390,8 +503,8 @@ header and call the :func:`import_modulename` function. The other functions can then be called and the extension types used as usual. -If the C code wanting to use these functions is part of more than one shared -library or executable, then :func:`import_modulename` function needs to be +If the C code wanting to use these functions is part of more than one shared +library or executable, then :func:`import_modulename` function needs to be called in each of the shared libraries which use these functions. If you crash with a segmentation fault (SIGSEGV on linux) when calling into one of these api calls, this is likely an indication that the shared library which @@ -399,30 +512,12 @@ the :func:`import_modulename` function before the api call which crashes. Any public C type or extension type declarations in the Cython module are also -made available when you include :file:`modulename_api.h`.:: +made available when you include :file:`modulename_api.h`.: - # delorean.pyx - cdef public struct Vehicle: - int speed - float power - - cdef api void activate(Vehicle *v): - if v.speed >= 88 and v.power >= 1.21: - print "Time travel achieved" - -.. sourcecode:: c - - # marty.c - #include "delorean_api.h" - - Vehicle car; - - int main(int argc, char *argv[]) { - import_delorean(); - car.speed = atoi(argv[1]); - car.power = atof(argv[2]); - activate(&car); - } +.. literalinclude:: ../../examples/userguide/external_C_code/delorean.pyx + +.. literalinclude:: ../../examples/userguide/external_C_code/marty.c + :language: C .. note:: @@ -434,12 +529,15 @@ Using the :keyword:`api` method does not require the C code using the declarations to be linked with the extension module in any way, as the Python import machinery is used to make the connection dynamically. However, only -functions can be accessed this way, not variables. +functions can be accessed this way, not variables. Note also that for the +module import mechanism to be set up correctly, the user must call +Py_Initialize() and Py_Finalize(); if you experience a segmentation fault in +the call to :func:`import_modulename`, it is likely that this wasn't done. You can use both :keyword:`public` and :keyword:`api` on the same function to make it available by both methods, e.g.:: - cdef public api void belt_and_braces(): + cdef public api void belt_and_braces() except *: ... However, note that you should include either :file:`modulename.h` or @@ -464,8 +562,8 @@ example,:: cdef public api: - void order_spam(int tons) - char *get_lunch(float tomato_size) + void order_spam(int tons) except * + char *get_lunch(float tomato_size) except NULL This can be a useful thing to do in a ``.pxd`` file (see :ref:`sharing-declarations`) to make the module's public interface @@ -475,7 +573,7 @@ --------------------------------- Cython provides facilities for acquiring and releasing the -`Global Interpreter Lock (GIL) `_. +`Global Interpreter Lock (GIL) `_. This may be useful when calling from multi-threaded code into (external C) code that may block, or when wanting to use Python from a (native) C thread callback. Releasing the GIL should @@ -500,9 +598,17 @@ with nogil: -Code in the body of the statement must not manipulate Python objects in any -way, and must not call anything that manipulates Python objects without first -re-acquiring the GIL. Cython currently does not check this. +Code in the body of the with-statement must not manipulate Python objects +in any way, and must not call anything that manipulates Python objects without +first re-acquiring the GIL. Cython validates these operations at compile time, +but cannot look into external C functions, for example. They must be correctly +declared as requiring or not requiring the GIL (see below) in order to make +Cython's checks effective. + +Since Cython 3.0, some simple Python statements can be used inside of ``nogil`` +sections: ``raise``, ``assert`` and ``print`` (the Py2 statement, not the function). +Since they tend to be lone Python statements, Cython will automatically acquire +and release the GIL around them for convenience. .. _gil: @@ -519,7 +625,7 @@ If the callback may be called from another non-Python thread, care must be taken to initialize the GIL first, through a call to -`PyEval_InitThreads() `_. +`PyEval_InitThreads() `_. If you're already using :ref:`cython.parallel ` in your module, this will already have been taken care of. The GIL may also be acquired through the ``with gil`` statement:: @@ -527,6 +633,26 @@ with gil: +.. _gil_conditional: + +Conditional Acquiring / Releasing the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Sometimes it is helpful to use a condition to decide whether to run a +certain piece of code with or without the GIL. This code would run anyway, +the difference is whether the GIL will be held or released. +The condition must be constant (at compile time). + +This could be useful for profiling, debugging, performance testing, and +for fused types (see :ref:`fused_gil_conditional`).:: + + DEF FREE_GIL = True + + with nogil(FREE_GIL): + + + with gil(False): + + Declaring a function as callable without the GIL -------------------------------------------------- diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/fusedtypes.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/fusedtypes.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/fusedtypes.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/fusedtypes.rst 2022-03-24 10:16:46.000000000 +0000 @@ -12,9 +12,7 @@ programming`_ and are akin to templates in C++ or generics in languages like Java / C#. -.. _generic programming: http://en.wikipedia.org/wiki/Generic_programming - -.. Note:: Support is still somewhat experimental, there may be bugs! +.. _generic programming: https://en.wikipedia.org/wiki/Generic_programming .. Note:: Fused types are not currently supported as attributes of extension types. Only variables and function/method arguments can be declared @@ -24,25 +22,7 @@ Quickstart ========== -:: - - cimport cython - - ctypedef fused char_or_float: - cython.char - cython.float - - - cpdef char_or_float plus_one(char_or_float var): - return var + 1 - - - def show_me(): - cdef: - cython.char a = 127 - cython.float b = 127 - print 'char', plus_one(a) - print 'float', plus_one(b) +.. literalinclude:: ../../examples/userguide/fusedtypes/char_or_float.pyx This gives:: @@ -83,24 +63,40 @@ cdef cfunc(my_fused_type arg): return arg + 1 -If the you use the same fused type more than once in an argument list, then each -specialization of the fused type must be the same:: +If the same fused type appears more than once in the function arguments, +then they will all have the same specialised type:: cdef cfunc(my_fused_type arg1, my_fused_type arg2): - return cython.typeof(arg1) == cython.typeof(arg2) + # arg1 and arg2 always have the same type here + return arg1 + arg2 -In this case, the type of both parameters is either an int, or a double -(according to the previous examples). However, because these arguments use the -same fused type ``my_fused_type``, both ``arg1`` and ``arg2`` are -specialized to the same type. Therefore this function returns True for every -possible valid invocation. You are allowed to mix fused types however:: +Here, the type of both parameters is either an int, or a double +(according to the previous examples), because they use the same fused type +name ``my_fused_type``. Mixing different fused types (or differently named +fused types) in the arguments will specialise them independently:: def func(A x, B y): ... -where ``A`` and ``B`` are different fused types. This will result in -specialized code paths for all combinations of types contained in ``A`` -and ``B``. +This will result in specialized code paths for all combinations of types +contained in ``A`` and ``B``, e.g.:: + + ctypedef fused my_fused_type: + cython.int + cython.double + + ctypedef fused my_fused_type2: + cython.int + cython.double + + cdef func(my_fused_type a, my_fused_type2 b): + # a and b may have the same or different types here + print("SAME!" if my_fused_type is my_fused_type2 else "NOT SAME!) + return a + b + +Note that a simple typedef to rename the fused type does not currently work here. +See Github issue :issue:`4302`. + Fused types and arrays ---------------------- @@ -267,7 +263,35 @@ long_pointer = &i if bunch_of_types in string_t: - print "s is a string!" + print("s is a string!") + +.. _fused_gil_conditional: + +Conditional GIL Acquiring / Releasing +===================================== + +Acquiring and releasing the GIL can be controlled by a condition +which is known at compile time (see :ref:`gil_conditional`). + +This is most useful when combined with fused types. +A fused type function may have to handle both cython native types +(e.g. cython.int or cython.double) and python types (e.g. object or bytes). +Conditional Acquiring / Releasing the GIL provides a method for running +the same piece of code either with the GIL released (for cython native types) +and with the GIL held (for python types).:: + + cimport cython + + ctypedef fused double_or_object: + cython.double + object + + def increment(double_or_object x): + with nogil(double_or_object is cython.double): + # Same code handles both cython.double (GIL is released) + # and python object (GIL is not released). + x = x + 1 + return x __signatures__ ============== diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/glossary.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/glossary.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/glossary.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/glossary.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,50 @@ +Glossary +======== + +.. glossary:: + + Extension type + "Extension type" can refer to either a Cython class defined with ``cdef class`` or ``@cclass``, + or more generally to any Python type that is ultimately implemented as a + native C struct (including the built-in types like `int` or `dict`). + + Dynamic allocation or Heap allocation + A C variable allocated with ``malloc`` (in C) or ``new`` (in C++) is + `allocated dynamically/heap allocated `_. + Its lifetime is until the user deletes it explicitly (with ``free`` in C or ``del`` in C++). + This can happen in a different function than the allocation. + + pointer + A **pointer** is a variable that stores the address of another variable + (i.e. direct address of the memory location). They allow for + dynamic memory allocation and deallocation. They can be used to build + dynamic data structures. + `Read more `__. + + Python object + When using Python, the contents of every variable is a Python object + (including Cython extension types). Key features of Python objects are that + they are passed *by reference* and that their lifetime is *managed* automatically + so that they are destroyed when no more references exist to them. + In Cython, they are distinct from C types, which are passed *by value* and whose + lifetime is managed depending on whether they are allocated on the stack or heap. + To explicitly declare a Python object variable in Cython use ``cdef object abc``. + Internally in C, they are referred to as ``PyObject*``. + + Stack allocation + A C variable declared within a function as ``cdef SomeType a`` + is said to be allocated on the stack. + It exists for the duration of the function only. + + Typed memoryview + A useful Cython type for getting quick access to blocks of memory. + A memoryview alone does not actually own any memory. + However, it can be initialized with a Python object that supports the + `buffer protocol`_ (typically "array" types, for example a Numpy array). + The memoryview keeps a reference to that Python object alive + and provides quick access to the memory without needing to go + through the Python API of the object and its + :meth:`__getitem__` / :meth:`__setitem__` methods. + For more information, see :ref:`memoryviews`. + +.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/index.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/index.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/index.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/index.rst 2022-03-24 10:16:46.000000000 +0000 @@ -16,6 +16,7 @@ wrapping_CPlusPlus fusedtypes pypy + migrating_to_cy30 limitations pyrex_differences memoryviews @@ -23,6 +24,7 @@ parallelism debugging numpy_tutorial + numpy_pythran Indices and tables ------------------ @@ -30,6 +32,3 @@ * :ref:`genindex` * :ref:`modindex` * :ref:`search` - -.. toctree:: - diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/language_basics.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/language_basics.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/language_basics.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/language_basics.rst 2022-03-24 10:16:46.000000000 +0000 @@ -11,81 +11,313 @@ Language Basics ***************** +.. include:: + ../two-syntax-variants-used + + +.. _declaring_data_types: + +Declaring Data Types +==================== + +As a dynamic language, Python encourages a programming style of considering +classes and objects in terms of their methods and attributes, more than where +they fit into the class hierarchy. + +This can make Python a very relaxed and comfortable language for rapid +development, but with a price - the 'red tape' of managing data types is +dumped onto the interpreter. At run time, the interpreter does a lot of work +searching namespaces, fetching attributes and parsing argument and keyword tuples. +This run-time ‘late binding’ is a major cause of Python’s relative slowness +compared to ‘early binding’ languages such as C++. + +However with Cython it is possible to gain significant speed-ups through +the use of ‘early binding’ programming techniques. + +.. note:: Typing is not a necessity + + Providing static typing to parameters and variables is convenience to + speed up your code, but it is not a necessity. Optimize where and when needed. + In fact, typing can *slow down* your code in the case where the + typing does not allow optimizations but where Cython still needs to + check that the type of some object matches the declared type. + + +.. _c_variable_and_type_definitions: + C variable and type definitions =============================== -The :keyword:`cdef` statement is used to declare C variables, either local or -module-level:: +C variables can be declared by + +* using the Cython specific :keyword:`cdef` statement, +* using PEP-484/526 type annotations with C data types or +* using the function ``cython.declare()``. + +The :keyword:`cdef` statement and ``declare()`` can define function-local and +module-level variables as well as attributes in classes, but type annotations only +affect local variables and attributes and are ignored at the module level. +This is because type annotations are not Cython specific, so Cython keeps +the variables in the module dict (as Python values) instead of making them +module internal C variables. Use ``declare()`` in Python code to explicitly +define global C variables. + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + a_global_variable = declare(cython.int) + + def func(): + i: cython.int + j: cython.int + k: cython.int + f: cython.float + g: cython.int[42] + h: cython.p_float + + i = j = 5 + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int a_global_variable + + def func(): + cdef int i, j, k + cdef float f, g[42], *h + + i = j = 5 + +As known from C, declared global variables are automatically initialised to +``0``, ``NULL`` or ``None``, depending on their type. However, also as known +from both Python and C, for a local variable, simply declaring it is not enough +to initialise it. If you use a local variable but did not assign a value, both +Cython and the C compiler will issue a warning "local variable ... referenced +before assignment". You need to assign a value at some point before first +using the variable, but you can also assign a value directly as part of +the declaration in most cases: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + a_global_variable = declare(cython.int, 42) + + def func(): + i: cython.int = 10 + f: cython.float = 2.5 + g: cython.int[4] = [1, 2, 3, 4] + h: cython.p_float = cython.address(f) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int a_global_variable + + def func(): + cdef int i = 10, j, k + cdef float f = 2.5 + # cdef float g[4] = [1,2,3,4] # currently not supported + cdef float *g = [1, 2, 3, 4] + cdef float *h = &f + +In addition to the basic types, C :keyword:`struct`, :keyword:`union` and :keyword:`enum` +are supported: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/struct_union_enum.py + + .. note:: Currently, Pure Python mode does not support enums. (GitHub issue :issue:`4252`) + + .. group-tab:: Cython - cdef int i, j, k - cdef float f, g[42], *h + .. literalinclude:: ../../examples/userguide/language_basics/struct_union_enum.pyx -and C :keyword:`struct`, :keyword:`union` or :keyword:`enum` types:: + See also :ref:`struct-union-enum-styles` - cdef struct Grail: - int age - float volume + .. note:: - cdef union Food: - char *spam - float *eggs + Structs can be declared as ``cdef packed struct``, which has + the same effect as the C directive ``#pragma pack(1)``. - cdef enum CheeseType: - cheddar, edam, - camembert + Declaring an enum as ``cpdef`` will create a :pep:`435`-style Python wrapper:: - cdef enum CheeseState: - hard = 1 - soft = 2 - runny = 3 + cpdef enum CheeseState: + hard = 1 + soft = 2 + runny = 3 -See also :ref:`struct-union-enum-styles` + There is currently no special syntax for defining a constant, but you can use + an anonymous :keyword:`enum` declaration for this purpose, for example,:: -There is currently no special syntax for defining a constant, but you can use -an anonymous :keyword:`enum` declaration for this purpose, for example,:: + cdef enum: + tons_of_spam = 3 - cdef enum: - tons_of_spam = 3 + .. note:: + the words ``struct``, ``union`` and ``enum`` are used only when + defining a type, not when referring to it. For example, to declare a variable + pointing to a ``Grail`` struct, you would write:: + + cdef Grail *gp + + and not:: + + cdef struct Grail *gp # WRONG .. note:: - the words ``struct``, ``union`` and ``enum`` are used only when - defining a type, not when referring to it. For example, to declare a variable - pointing to a ``Grail`` you would write:: - cdef Grail *gp + There is also support for giving names to types using the + ``ctypedef`` statement or the ``cython.typedef()`` function, e.g. + + .. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + ULong = cython.typedef(cython.ulong) + + IntPtr = cython.typedef(cython.p_int) + + .. group-tab:: Cython + + .. code-block:: cython + + ctypedef unsigned long ULong + + ctypedef int* IntPtr + + +You can create a C function by declaring it with :keyword:`cdef` or by decorating a Python function with ``@cfunc``: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def eggs(l: cython.ulong, f: cython.float) -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython - and not:: + cdef int eggs(unsigned long l, float f): + ... - cdef struct Grail *gp # WRONG +You can read more about them in :ref:`python_functions_vs_c_functions`. - There is also a ``ctypedef`` statement for giving names to types, e.g.:: +Classes can be declared as :ref:`extension-types`. Those will +have a behavior very close to python classes, but are faster because they use a ``struct`` +internally to store attributes. +They are declared with the :keyword:`cdef` keyword or the ``@cclass`` class decorator. - ctypedef unsigned long ULong +Here is a simple example: - ctypedef int* IntPtr +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx + +You can read more about them in :ref:`extension-types`. + + +.. _typing_types: +.. _types: Types ----- -Cython uses the normal C syntax for C types, including pointers. It provides +The Cython language uses the normal C syntax for C types, including pointers. It provides all the standard C types, namely ``char``, ``short``, ``int``, ``long``, -``long long`` as well as their ``unsigned`` versions, e.g. ``unsigned int``. +``long long`` as well as their ``unsigned`` versions, +e.g. ``unsigned int`` (``cython.uint`` in Python code). The special ``bint`` type is used for C boolean values (``int`` with 0/non-0 values for False/True) and ``Py_ssize_t`` for (signed) sizes of Python containers. -Pointer types are constructed as in C, by appending a ``*`` to the base type -they point to, e.g. ``int**`` for a pointer to a pointer to a C int. -Arrays use the normal C array syntax, e.g. ``int[10]``. Note that Cython uses -array access for pointer dereferencing, as ``*x`` is not valid Python syntax, +Pointer types are constructed as in C when using Cython syntax, by appending a ``*`` to the base type +they point to, e.g. ``int**`` for a pointer to a pointer to a C int. In Pure python mode, simple pointer types +use a naming scheme with "p"s instead, separated from the type name with an underscore, e.g. ``cython.pp_int`` for a pointer to +a pointer to a C int. Further pointer types can be constructed with the ``cython.pointer()`` function, +e.g. ``cython.pointer(cython.int)``. + + +Arrays use the normal C array syntax, e.g. ``int[10]``, and the size must be known +at compile time for stack allocated arrays. Cython doesn't support variable length arrays from C99. +Note that Cython uses array access for pointer dereferencing, as ``*x`` is not valid Python syntax, whereas ``x[0]`` is. Also, the Python types ``list``, ``dict``, ``tuple``, etc. may be used for -static typing, as well as any user defined extension types. The Python types -int, long, and float are not available for static typing and instead interpreted as C -``int``, ``long``, and ``float`` respectively, as statically typing variables with these Python -types has zero advantages. +static typing, as well as any user defined :ref:`extension-types`. +For example + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + foo: list = [] + + .. group-tab:: Cython + + .. code-block:: cython + + cdef list foo = [] + +This requires an *exact* match of the class, it does not allow subclasses. +This allows Cython to optimize code by accessing internals of the builtin class, +which is the main reason for declaring builtin types in the first place. + +For declared builtin types, Cython uses internally a C variable of type ``PyObject*``. + +.. note:: The Python types ``int``, ``long``, and ``float`` are not available for static + typing in ``.pyx`` files and instead interpreted as C ``int``, ``long``, and ``float`` + respectively, as statically typing variables with these Python + types has zero advantages. On the other hand, annotating in Pure Python with + ``int``, ``long``, and ``float`` Python types will be interpreted as + Python object types. + +Cython provides an accelerated and typed equivalent of a Python tuple, the ``ctuple``. +A ``ctuple`` is assembled from any valid C types. For example + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + bar: (cython.double, cython.int) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef (double, int) bar + +They compile down to C-structures and can be used as efficient alternatives to +Python tuples. + While these C types can be vastly faster, they have C semantics. Specifically, the integer types overflow and the C ``float`` type only has 32 bits of precision @@ -94,67 +326,124 @@ If you want to use these numeric Python types simply omit the type declaration and let them be objects. +It is also possible to declare :ref:`extension-types` (declared with ``cdef class`` or the ``@cclass`` decorator). +This does allow subclasses. This typing is mostly used to access +``cdef``/``@cfunc`` methods and attributes of the extension type. +The C code uses a variable which is a pointer to a structure of the +specific type, something like ``struct MyExtensionTypeObject*``. + Grouping multiple C declarations -------------------------------- If you have a series of declarations that all begin with :keyword:`cdef`, you -can group them into a :keyword:`cdef` block like this:: +can group them into a :keyword:`cdef` block like this: + +.. note:: This is supported only in Cython's ``cdef`` syntax. - cdef: - struct Spam: - int tons - - int i - float f - Spam *p +.. literalinclude:: ../../examples/userguide/language_basics/cdef_block.pyx - void f(Spam *s): - print s.tons, "Tons of spam" +.. _cpdef: +.. _cdef: +.. _python_functions_vs_c_functions: Python functions vs. C functions ================================== There are two kinds of function definition in Cython: -Python functions are defined using the def statement, as in Python. They take -Python objects as parameters and return Python objects. +Python functions are defined using the :keyword:`def` statement, as in Python. They take +:term:`Python objects` as parameters and return Python objects. -C functions are defined using the new :keyword:`cdef` statement. They take +C functions are defined using the :keyword:`cdef` statement in Cython syntax or with the ``@cfunc`` decorator. They take either Python objects or C values as parameters, and can return either Python objects or C values. Within a Cython module, Python functions and C functions can call each other freely, but only Python functions can be called from outside the module by interpreted Python code. So, any functions that you want to "export" from your -Cython module must be declared as Python functions using def. -There is also a hybrid function, called :keyword:`cpdef`. A :keyword:`cpdef` -can be called from anywhere, but uses the faster C calling conventions -when being called from other Cython code. A :keyword:`cpdef` can also be overridden +Cython module must be declared as Python functions using ``def``. +There is also a hybrid function, declared with :keyword:`cpdef` in ``.pyx`` +files or with the ``@ccall`` decorator. These functions +can be called from anywhere, but use the faster C calling convention +when being called from other Cython code. They can also be overridden by a Python method on a subclass or an instance attribute, even when called from Cython. If this happens, most performance gains are of course lost and even if it does not, -there is a tiny overhead in calling a :keyword:`cpdef` method from Cython compared to -calling a :keyword:`cdef` method. +there is a tiny overhead in calling such a method from Cython compared to +calling a C method. Parameters of either type of function can be declared to have C data types, -using normal C declaration syntax. For example,:: +using normal C declaration syntax. For example, - def spam(int i, char *s): - ... +.. tabs:: - cdef int eggs(unsigned long l, float f): - ... + .. group-tab:: Pure Python + + .. code-block:: python + + def spam(i: cython.int, s: cython.p_char): + ... + + @cython.cfunc + def eggs(l: cython.ulong, f: cython.float) -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + def spam(int i, char *s): + ... + + + cdef int eggs(unsigned long l, float f): + ... + +``ctuples`` may also be used + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def chips(t: (cython.long, cython.long, cython.double)) -> (cython.int, cython.float): + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef (int, float) chips((long, long, double) t): + ... When a parameter of a Python function is declared to have a C data type, it is passed in as a Python object and automatically converted to a C value, if possible. In other words, the definition of ``spam`` above is equivalent to -writing:: +writing + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def spam(python_i, python_s): + i: cython.int = python_i + s: cython.p_char = python_s + ... + + .. group-tab:: Cython + + .. code-block:: cython + + def spam(python_i, python_s): + cdef int i = python_i + cdef char* s = python_s + ... - def spam(python_i, python_s): - cdef int i = python_i - cdef char* s = python_s - ... Automatic conversion is currently only possible for numeric types, string types and structs (composed recursively of any of these types); @@ -167,116 +456,333 @@ C functions, on the other hand, can have parameters of any type, since they're passed in directly using a normal C function call. -Functions declared using :keyword:`cdef`, like Python functions, will return a :keyword:`False` +C Functions declared using :keyword:`cdef` or the ``@cfunc`` decorator with a +Python object return type, like Python functions, will return a :keyword:`None` value when execution leaves the function body without an explicit return value. This is in contrast to C/C++, which leaves the return value undefined. +In the case of non-Python object return types, the equivalent of zero is returned, for example, 0 for ``int``, :keyword:`False` for ``bint`` and :keyword:`NULL` for pointer types. A more complete comparison of the pros and cons of these different method types can be found at :ref:`early-binding-for-speed`. + Python objects as parameters and return values ---------------------------------------------- If no type is specified for a parameter or return value, it is assumed to be a -Python object. (Note that this is different from the C convention, where it -would default to int.) For example, the following defines a C function that -takes two Python objects as parameters and returns a Python object:: +Python object. (Note that this is different from the C convention, where it +would default to ``int``.) For example, the following defines a C function that +takes two Python objects as parameters and returns a Python object - cdef spamobjs(x, y): - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def spamobjs(x, y): + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef spamobjs(x, y): + ... Reference counting for these objects is performed automatically according to the standard Python/C API rules (i.e. borrowed references are taken as parameters and a new reference is returned). -The name object can also be used to explicitly declare something as a Python + .. warning:: + + This only applies to Cython code. Other Python packages which + are implemented in C like NumPy may not follow these conventions. + +The type name ``object`` can also be used to explicitly declare something as a Python object. This can be useful if the name being declared would otherwise be taken -as the name of a type, for example,:: +as the name of a type, for example, - cdef ftang(object int): - ... +.. tabs:: -declares a parameter called int which is a Python object. You can also use -object as the explicit return type of a function, e.g.:: + .. group-tab:: Pure Python - cdef object ftang(object int): - ... + .. code-block:: python + + @cython.cfunc + def ftang(int: object): + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef ftang(object int): + ... + +declares a parameter called ``int`` which is a Python object. You can also use +``object`` as the explicit return type of a function, e.g. + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def ftang(int: object) -> object: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef object ftang(object int): + ... In the interests of clarity, it is probably a good idea to always be explicit about object parameters in C functions. -Error return values -------------------- +To create a borrowed reference, specify the parameter type as ``PyObject*``. +Cython won't perform automatic ``Py_INCREF``, or ``Py_DECREF``, e.g.: -If you don't do anything special, a function declared with :keyword:`cdef` that -does not return a Python object has no way of reporting Python exceptions to -its caller. If an exception is detected in such a function, a warning message -is printed and the exception is ignored. - -If you want a C function that does not return a Python object to be able to -propagate exceptions to its caller, you need to declare an exception value for -it. Here is an example:: +.. tabs:: - cdef int spam() except -1: - ... + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/parameter_refcount.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/parameter_refcount.pyx + +will display:: + + Initial refcount: 2 + Inside owned_reference: 3 + Inside borrowed_reference: 2 + + +.. _optional_arguments: + +Optional Arguments +------------------ + +Unlike C, it is possible to use optional arguments in C and ``cpdef``/``@ccall`` functions. +There are differences though whether you declare them in a ``.pyx``/``.py`` +file or the corresponding ``.pxd`` file. + +To avoid repetition (and potential future inconsistencies), default argument values are +not visible in the declaration (in ``.pxd`` files) but only in +the implementation (in ``.pyx`` files). + +When in a ``.pyx``/``.py`` file, the signature is the same as it is in Python itself: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.py + :caption: optional_subclassing.py + + .. group-tab:: Cython -With this declaration, whenever an exception occurs inside spam, it will -immediately return with the value ``-1``. Furthermore, whenever a call to spam -returns ``-1``, an exception will be assumed to have occurred and will be -propagated. - -When you declare an exception value for a function, you should never -explicitly or implicitly return that value. In particular, if the exceptional return value -is a ``False`` value, then you should ensure the function will never terminate via an implicit -or empty return. + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx + :caption: optional_subclassing.pyx + +When in a ``.pxd`` file, the signature is different like this example: ``cdef foo(x=*)``. +This is because the program calling the function just needs to know what signatures are +possible in C, but doesn't need to know the value of the default arguments.: + +.. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pxd + :caption: optional_subclassing.pxd + +.. note:: + The number of arguments may increase when subclassing, + but the arg types and order must be the same, as shown in the example above. + +There may be a slight performance penalty when the optional arg is overridden +with one that does not have default values. + + +.. _keyword_only_argument: + +Keyword-only Arguments +---------------------- + +As in Python 3, ``def`` functions can have keyword-only arguments +listed after a ``"*"`` parameter and before a ``"**"`` parameter if any: + +.. literalinclude:: ../../examples/userguide/language_basics/kwargs_1.pyx + +As shown above, the ``c``, ``d`` and ``e`` arguments can not be +passed as positional arguments and must be passed as keyword arguments. +Furthermore, ``c`` and ``e`` are **required** keyword arguments +since they do not have a default value. + +A single ``"*"`` without argument name can be used to +terminate the list of positional arguments: + +.. literalinclude:: ../../examples/userguide/language_basics/kwargs_2.pyx + +Shown above, the signature takes exactly two positional +parameters and has two required keyword parameters. + + +Function Pointers +----------------- + +Functions declared in a ``struct`` are automatically converted to function pointers. + +For using error return values with function pointers, see the note at the bottom +of :ref:`error_return_values`. + + +.. _error_return_values: + +Error return values +------------------- + +In Python (more specifically, in the CPython runtime), exceptions that occur +inside of a function are signaled to the caller and propagated up the call stack +through defined error return values. For functions that return a Python object +(and thus, a pointer to such an object), the error return value is simply the +``NULL`` pointer, so any function returning a Python object has a well-defined +error return value. + +While this is always the case for C functions, functions +defined as C functions or ``cpdef``/``@ccall`` functions can return arbitrary C types, +which do not have such a well-defined error return value. Thus, if an +exception is detected in such a function, a warning message is printed, +the exception is ignored, and the function returns immediately without +propagating the exception to its caller. + +If you want such a C function to be able to propagate exceptions, you need +to declare an exception return value for it as a contract with the caller. +Here is an example + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1) + def spam() -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except -1: + ... + +With this declaration, whenever an exception occurs inside ``spam``, it will +immediately return with the value ``-1``. From the caller's side, whenever +a call to spam returns ``-1``, the caller will assume that an exception has +occurred and can now process or propagate it. + +When you declare an exception value for a function, you should never explicitly +or implicitly return that value. This includes empty :keyword:`return` +statements, without a return value, for which Cython inserts the default return +value (e.g. ``0`` for C number types). In general, exception return values +are best chosen from invalid or very unlikely return values of the function, +such as a negative value for functions that return only non-negative results, +or a very large value like ``INT_MAX`` for a function that "usually" only +returns small results. If all possible return values are legal and you can't reserve one entirely for signalling errors, you can use an alternative -form of exception value declaration:: +form of exception value declaration - cdef int spam() except? -1: - ... +.. tabs:: -The "?" indicates that the value ``-1`` only indicates a possible error. In this -case, Cython generates a call to :c:func:`PyErr_Occurred` if the exception value is -returned, to make sure it really is an error. + .. group-tab:: Pure Python -There is also a third form of exception value declaration:: + .. code-block:: python - cdef int spam() except *: - ... + @cython.cfunc + @cython.exceptval(-1, check=True) + def spam() -> cython.int: + ... + + The keyword argument ``check=True`` indicates that the value ``-1`` _may_ signal an error. + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except? -1: + ... + + The ``?`` indicates that the value ``-1`` _may_ signal an error. + +In this case, Cython generates a call to :c:func:`PyErr_Occurred` if the exception value +is returned, to make sure it really received an exception and not just a normal +result. + +There is also a third form of exception value declaration + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(check=True) + def spam() -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except *: + ... This form causes Cython to generate a call to :c:func:`PyErr_Occurred` after -every call to spam, regardless of what value it returns. If you have a -function returning void that needs to propagate errors, you will have to use -this form, since there isn't any return value to test. -Otherwise there is little use for this form. +*every* call to spam, regardless of what value it returns. If you have a +function returning ``void`` that needs to propagate errors, you will have to +use this form, since there isn't any error return value to test. +Otherwise, an explicit error return value allows the C compiler to generate +more efficient code and is thus generally preferable. An external C++ function that may raise an exception can be declared with:: cdef int spam() except + +.. note:: These declarations are not used in Python code, only in ``.pxd`` and ``.pyx`` files. + See :ref:`wrapping-cplusplus` for more details. Some things to note: -* Exception values can only declared for functions returning an integer, enum, - float or pointer type, and the value must be a constant expression. - Void functions can only use the ``except *`` form. +* Exception values can only be declared for functions returning a C integer, + enum, float or pointer type, and the value must be a constant expression. + Functions that return ``void``, or a struct/union by value, can only use + the ``except *`` or ``exceptval(check=True)`` form. * The exception value specification is part of the signature of the function. If you're passing a pointer to a function as a parameter or assigning it to a variable, the declared type of the parameter or variable must have - the same exception value specification (or lack thereof). Here is an - example of a pointer-to-function declaration with an exception - value:: + the same exception value specification (or lack thereof). Here is an + example of a pointer-to-function declaration with an exception value:: int (*grail)(int, char*) except -1 + .. note:: Pointers to functions are currently not supported by pure Python mode. (GitHub issue :issue:`4279`) + * You don't need to (and shouldn't) declare exception values for functions which return Python objects. Remember that a function with no declared - return type implicitly returns a Python object. (Exceptions on such functions - are implicitly propagated by returning NULL.) + return type implicitly returns a Python object. (Exceptions on such + functions are implicitly propagated by returning ``NULL``.) + + +.. _checking_return_values_of_non_cython_functions: Checking return values of non-Cython functions ---------------------------------------------- @@ -292,13 +798,56 @@ for propagating Python exceptions that have already been raised, either by a Cython function or a C function that calls Python/C API routines. To get an exception from a non-Python-aware function such as :func:`fopen`, you will have to check the -return value and raise it yourself, for example,:: +return value and raise it yourself, for example: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/open_file.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/open_file.pyx + + +.. _overriding_in_extension_types: + +Overriding in extension types +----------------------------- + + +``cpdef``/``@ccall`` methods can override C methods: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx - cdef FILE* p - p = fopen("spam.txt", "r") - if p == NULL: - raise SpamError("Couldn't open the spam file") +When subclassing an extension type with a Python class, +Python methods can override ``cpdef``/``@ccall`` methods but not plain C methods: +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/override.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/override.pyx + +If ``C`` above would be an extension type (``cdef class``), +this would not work correctly. +The Cython compiler will give a warning in that case. + + +.. _type-conversion: Automatic type conversions ========================== @@ -323,6 +872,8 @@ +----------------------------+--------------------+------------------+ | char* | str/bytes | str/bytes [#]_ | +----------------------------+--------------------+------------------+ +| C array | iterable | list [#2]_ | ++----------------------------+--------------------+------------------+ | struct, | | dict [#1]_ | | union | | | +----------------------------+--------------------+------------------+ @@ -333,7 +884,11 @@ a value for each of the union fields. Cython 0.23 and later, however, will refuse to automatically convert a union with unsafe type combinations. An example is a union of an ``int`` and a ``char*``, - in which case the pointer value may or be not be a valid pointer. + in which case the pointer value may or may not be a valid pointer. + +.. [#2] Other than signed/unsigned char[]. + The conversion will fail if the length of C array is not known at compile time, + and when using a slice of a C array. Caveats when using a Python string in a C context @@ -347,13 +902,27 @@ live long enough, you will need to copy the C string. Cython detects and prevents some mistakes of this kind. For instance, if you -attempt something like:: +attempt something like + +.. tabs:: - cdef char *s - s = pystring1 + pystring2 + .. group-tab:: Pure Python -then Cython will produce the error message ``Obtaining char* from temporary -Python value``. The reason is that concatenating the two Python strings + .. code-block:: python + + def main(): + s: cython.p_char + s = pystring1 + pystring2 + + .. group-tab:: Cython + + .. code-block:: cython + + cdef char *s + s = pystring1 + pystring2 + +then Cython will produce the error message ``Storing unsafe C derivative of temporary +Python reference``. The reason is that concatenating the two Python strings produces a new Python string object that is referenced only by a temporary internal variable that Cython generates. As soon as the statement has finished, the temporary variable will be decrefed and the Python string deallocated, @@ -361,11 +930,26 @@ compile it. The solution is to assign the result of the concatenation to a Python -variable, and then obtain the ``char*`` from that, i.e.:: +variable, and then obtain the ``char*`` from that, i.e. + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + s: cython.p_char + p = pystring1 + pystring2 + s = p + + .. group-tab:: Cython - cdef char *s - p = pystring1 + pystring2 - s = p + .. code-block:: cython + + cdef char *s + p = pystring1 + pystring2 + s = p It is then your responsibility to hold the reference p for as long as necessary. @@ -375,6 +959,108 @@ detect a problem that exists. Ultimately, you need to understand the issue and be careful what you do. + +.. _type_casting: + +Type Casting +------------ + +The Cython language supports type casting in a similar way as C. Where C uses ``"("`` and ``")"``, +Cython uses ``"<"`` and ``">"``. In pure python mode, the ``cython.cast()`` function is used. For example: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + p: cython.p_char + q: cython.p_float + p = cython.cast(cython.p_char, q) + + When casting a C value to a Python object type or vice versa, + Cython will attempt a coercion. Simple examples are casts like ``cast(int, pyobj_value)``, + which convert a Python number to a plain C ``int`` value, or the statement ``cast(bytes, charptr_value)``, + which copies a C ``char*`` string into a new Python bytes object. + + .. note:: Cython will not prevent a redundant cast, but emits a warning for it. + + To get the address of some Python object, use a cast to a pointer type + like ``cast(p_void, ...)`` or ``cast(pointer(PyObject), ...)``. + You can also cast a C pointer back to a Python object reference + with ``cast(object, ...)``, or to a more specific builtin or extension type + (e.g. ``cast(MyExtType, ptr)``). This will increase the reference count of + the object by one, i.e. the cast returns an owned reference. + Here is an example: + + + .. group-tab:: Cython + + .. code-block:: cython + + cdef char *p + cdef float *q + p = q + + When casting a C value to a Python object type or vice versa, + Cython will attempt a coercion. Simple examples are casts like ``pyobj_value``, + which convert a Python number to a plain C ``int`` value, or the statement ``charptr_value``, + which copies a C ``char*`` string into a new Python bytes object. + + .. note:: Cython will not prevent a redundant cast, but emits a warning for it. + + To get the address of some Python object, use a cast to a pointer type + like ```` or ````. + You can also cast a C pointer back to a Python object reference + with ````, or to a more specific builtin or extension type + (e.g. ``ptr``). This will increase the reference count of + the object by one, i.e. the cast returns an owned reference. + Here is an example: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.pxd + :caption: casting_python.pxd + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.py + :caption: casting_python.py + + Casting with ``cast(object, ...)`` creates an owned reference. Cython will automatically + perform a ``Py_INCREF`` and ``Py_DECREF`` operation. Casting to + ``cast(pointer(PyObject), ...)`` creates a borrowed reference, leaving the refcount unchanged. + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.pyx + :caption: casting_python.pyx + + The precedence of ``<...>`` is such that ``a.b.c`` is interpreted as ``(a.b.c)``. + + Casting to ```` creates an owned reference. Cython will automatically + perform a ``Py_INCREF`` and ``Py_DECREF`` operation. Casting to + ```` creates a borrowed reference, leaving the refcount unchanged. + + +.. _checked_type_casts: + +Checked Type Casts +------------------ + +A cast like ``x`` or ``cast(MyExtensionType, x)`` will cast ``x`` to the class +``MyExtensionType`` without any checking at all. + +To have a cast checked, use ``x`` in Cython syntax +or ``cast(MyExtensionType, x, typecheck=True)``. +In this case, Cython will apply a runtime check that raises a ``TypeError`` +if ``x`` is not an instance of ``MyExtensionType``. +This tests for the exact class for builtin types, +but allows subclasses for :ref:`extension-types`. + + +.. _statements_and_expressions: + Statements and expressions ========================== @@ -390,6 +1076,7 @@ Python operations are automatically checked for errors, with appropriate action taken. + Differences between C and Cython expressions -------------------------------------------- @@ -399,17 +1086,38 @@ * An integer literal is treated as a C constant, and will be truncated to whatever size your C compiler thinks appropriate. - To get a Python integer (of arbitrary precision) cast immediately to - an object (e.g. ``100000000000000000000``). The ``L``, ``LL``, - and ``U`` suffixes have the same meaning as in C. + To get a Python integer (of arbitrary precision), cast immediately to + an object (e.g. ``100000000000000000000`` or ``cast(object, 100000000000000000000)``). The ``L``, ``LL``, + and ``U`` suffixes have the same meaning in Cython syntax as in C. * There is no ``->`` operator in Cython. Instead of ``p->x``, use ``p.x`` * There is no unary ``*`` operator in Cython. Instead of ``*p``, use ``p[0]`` -* There is an ``&`` operator, with the same semantics as in C. -* The null C pointer is called ``NULL``, not ``0`` (and ``NULL`` is a reserved word). -* Type casts are written ``value`` , for example,:: +* There is an ``&`` operator in Cython, with the same semantics as in C. + In pure python mode, use the ``cython.address()`` function instead. +* The null C pointer is called ``NULL``, not ``0``. ``NULL`` is a reserved word in Cython + and special object in pure python mode. +* Type casts are written ``value`` or ``cast(type, value)``, for example, + + .. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + p: cython.p_char + q: cython.p_float + + p = cython.cast(cython.p_char, q) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef char* p + cdef float* q + + p = q - cdef char* p, float* q - p = q Scope rules ----------- @@ -422,6 +1130,8 @@ always a Python object. +.. _built_in_functions: + Built-in Functions ------------------ @@ -487,9 +1197,12 @@ Keep in mind that there are some differences in operator precedence between Python and C, and that Cython uses the Python precedences, not the C ones. + Integer for-loops ------------------ +.. note:: This syntax is supported only in Cython files. Use a normal `for-in-range()` loop instead. + Cython recognises the usual Python for-in-range integer loop pattern:: for i in range(n): @@ -532,8 +1245,81 @@ body, and the loop may have an else clause. -The include statement -===================== +.. _cython_file_types: + +Cython file types +================= + +There are three file types in Cython: + +* The implementation files, carrying a ``.py`` or ``.pyx`` suffix. +* The definition files, carrying a ``.pxd`` suffix. +* The include files, carrying a ``.pxi`` suffix. + + +The implementation file +----------------------- + +The implementation file, as the name suggest, contains the implementation +of your functions, classes, extension types, etc. Nearly all the +python syntax is supported in this file. Most of the time, a ``.py`` +file can be renamed into a ``.pyx`` file without changing +any code, and Cython will retain the python behavior. + +It is possible for Cython to compile both ``.py`` and ``.pyx`` files. +The name of the file isn't important if one wants to use only the Python syntax, +and Cython won't change the generated code depending on the suffix used. +Though, if one want to use the Cython syntax, using a ``.pyx`` file is necessary. + +In addition to the Python syntax, the user can also +leverage Cython syntax (such as ``cdef``) to use C variables, can +declare functions as ``cdef`` or ``cpdef`` and can import C definitions +with :keyword:`cimport`. Many other Cython features usable in implementation files +can be found throughout this page and the rest of the Cython documentation. + +There are some restrictions on the implementation part of some :ref:`extension-types` +if the corresponding definition file also defines that type. + +.. note:: + + When a ``.pyx`` file is compiled, Cython first checks to see if a corresponding + ``.pxd`` file exists and processes it first. It acts like a header file for + a Cython ``.pyx`` file. You can put inside functions that will be used by + other Cython modules. This allows different Cython modules to use functions + and classes from each other without the Python overhead. To read more about + what how to do that, you can see :ref:`pxd_files`. + + +The definition file +------------------- + +A definition file is used to declare various things. + +Any C declaration can be made, and it can be also a declaration of a C variable or +function implemented in a C/C++ file. This can be done with ``cdef extern from``. +Sometimes, ``.pxd`` files are used as a translation of C/C++ header files +into a syntax that Cython can understand. This allows then the C/C++ variable and +functions to be used directly in implementation files with :keyword:`cimport`. +You can read more about it in :ref:`external-C-code` and :ref:`wrapping-cplusplus`. + +It can also contain the definition part of an extension type and the declarations +of functions for an external library. + +It cannot contain the implementations of any C or Python functions, or any +Python class definitions, or any executable statements. It is needed when one +wants to access :keyword:`cdef` attributes and methods, or to inherit from +:keyword:`cdef` classes defined in this module. + +.. note:: + + You don't need to (and shouldn't) declare anything in a declaration file + :keyword:`public` in order to make it available to other Cython modules; its mere + presence in a definition file does that. You only need a public + declaration if you want to make something available to external C code. + + +The include statement and include files +--------------------------------------- .. warning:: Historically the ``include`` statement was used for sharing declarations. @@ -544,12 +1330,14 @@ include "spamstuff.pxi" -The contents of the named file are textually included at that point. The +The contents of the named file are textually included at that point. The included file can contain any complete statements or declarations that are valid in the context where the include statement appears, including other -include statements. The contents of the included file should begin at an +include statements. The contents of the included file should begin at an indentation level of zero, and will be treated as though they were indented to -the level of the include statement that is including the file. +the level of the include statement that is including the file. The include +statement cannot, however, be used outside of the module scope, such as inside +of functions or class bodies. .. note:: @@ -558,12 +1346,27 @@ :ref:`sharing-declarations`. +.. _conditional_compilation: + Conditional Compilation ======================= Some features are available for conditional compilation and compile-time constants within a Cython source file. +.. note:: + + This feature has very little use cases. Specifically, it is not a good + way to adapt code to platform and environment. Use code generation or + (preferably) C compile time adaptation for this. See, for example, + :ref:`verbatim_c`. + +.. note:: + + Cython currently does not support conditional compilation and compile-time + definitions in Pure Python mode. As it stands, this is unlikely to change. + + Compile-Time Definitions ------------------------ @@ -600,11 +1403,8 @@ expression must evaluate to a Python value of type ``int``, ``long``, ``float``, ``bytes`` or ``unicode`` (``str`` in Py3). -:: +.. literalinclude:: ../../examples/userguide/language_basics/compile_time.pyx - cdef int a1[ArraySize] - cdef int a2[OtherArraySize] - print "I like", FavouriteFood Conditional Statements ---------------------- @@ -631,4 +1431,3 @@ expressions as for the ``DEF`` statement, although they can evaluate to any Python value, and the truth of the result is determined in the usual Python way. - diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/limitations.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/limitations.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/limitations.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/limitations.rst 2022-03-24 10:16:46.000000000 +0000 @@ -8,12 +8,12 @@ This page used to list bugs in Cython that made the semantics of compiled code differ from that in Python. Most of the missing -features have been fixed in Cython 0.15. The bug tracker has an -up-to-date `list of remaining compatibility issues`_. Note that a -future version 1.0 of Cython is planned to provide full Python -language compatibility. +features have been fixed in Cython 0.15. A future version of +Cython is planned to provide full Python language compatibility. +For now, the issue tracker can provide an overview of deviations +that we are aware of and would like to see fixed. -.. _`list of remaining compatibility issues`: http://trac.cython.org/cython_trac/query?status=assigned&status=new&status=reopened&component=Python+Semantics&component=Python3+Semantics&order=priority&col=id&col=summary&col=component&col=status&col=type&col=priority&col=milestone +https://github.com/cython/cython/labels/Python%20Semantics Below is a list of differences that we will probably not be addressing. Most of these things that fall more into the implementation details rather @@ -63,4 +63,4 @@ if some_runtime_expression: b = a # creates a new Python float object c = a # creates a new Python float object - print b is c # most likely not the same object + print(b is c) # most likely not the same object diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/memoryviews.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/memoryviews.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/memoryviews.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/memoryviews.rst 2022-03-24 10:16:46.000000000 +0000 @@ -20,6 +20,9 @@ class attribute, etc) and can be obtained from nearly any object that exposes writable buffer through the `PEP 3118`_ buffer interface. +.. _`PEP 3118`: https://www.python.org/dev/peps/pep-3118/ + + .. _view_quickstart: Quickstart @@ -28,60 +31,7 @@ If you are used to working with NumPy, the following examples should get you started with Cython memory views. -:: - - from cython.view cimport array as cvarray - import numpy as np - - # Memoryview on a NumPy array - narr = np.arange(27, dtype=np.dtype("i")).reshape((3, 3, 3)) - cdef int [:, :, :] narr_view = narr - - # Memoryview on a C array - cdef int carr[3][3][3] - cdef int [:, :, :] carr_view = carr - - # Memoryview on a Cython array - cyarr = cvarray(shape=(3, 3, 3), itemsize=sizeof(int), format="i") - cdef int [:, :, :] cyarr_view = cyarr - - # Show the sum of all the arrays before altering it - print("NumPy sum of the NumPy array before assignments: %s" % narr.sum()) - - # We can copy the values from one memoryview into another using a single - # statement, by either indexing with ... or (NumPy-style) with a colon. - carr_view[...] = narr_view - cyarr_view[:] = narr_view - # NumPy-style syntax for assigning a single value to all elements. - narr_view[:, :, :] = 3 - - # Just to distinguish the arrays - carr_view[0, 0, 0] = 100 - cyarr_view[0, 0, 0] = 1000 - - # Assigning into the memoryview on the NumPy array alters the latter - print("NumPy sum of NumPy array after assignments: %s" % narr.sum()) - - # A function using a memoryview does not usually need the GIL - cpdef int sum3d(int[:, :, :] arr) nogil: - cdef size_t i, j, k - cdef int total = 0 - I = arr.shape[0] - J = arr.shape[1] - K = arr.shape[2] - for i in range(I): - for j in range(J): - for k in range(K): - total += arr[i, j, k] - return total - - # A function accepting a memoryview knows how to use a NumPy array, - # a C array, a Cython array... - print("Memoryview sum of NumPy array is %s" % sum3d(narr)) - print("Memoryview sum of C array is %s" % sum3d(carr)) - print("Memoryview sum of Cython array is %s" % sum3d(cyarr)) - # ... and of course, a memoryview. - print("Memoryview sum of C memoryview is %s" % sum3d(carr_view)) +.. literalinclude:: ../../examples/userguide/memoryviews/quickstart.pyx This code should give the following output:: @@ -109,35 +59,31 @@ cdef int[:,:,:] view3D = exporting_object -A 2D view that restricts the first dimension of a buffer to 100 rows -starting at the second (index 1) and then skips every second (odd) row:: - - cdef int[1:102:2,:] partial_view = exporting_object - -This also works conveniently as function arguments: +They also work conveniently as function arguments: .. code-block:: cython - def process_3d_buffer(int[1:102:2,:] view not None): + def process_3d_buffer(int[:,:,:] view not None): ... The ``not None`` declaration for the argument automatically rejects None values as input, which would otherwise be allowed. The reason why None is allowed by default is that it is conveniently used for return -arguments:: +arguments: - def process_buffer(int[:,:] input not None, - int[:,:] output = None): - if output is None: - output = ... # e.g. numpy.empty_like(input) - # process 'input' into 'output' - return output +.. literalinclude:: ../../examples/userguide/memoryviews/not_none.pyx Cython will reject incompatible buffers automatically, e.g. passing a three dimensional buffer into a function that requires a two dimensional buffer will raise a ``ValueError``. +To use a memory view on a numpy array with a custom dtype, you'll need to +declare an equivalent packed struct that mimics the dtype: + +.. literalinclude:: ../../examples/userguide/memoryviews/custom_dtype.pyx + + Indexing -------- @@ -155,40 +101,24 @@ print(buf[-1,-2]) The following function loops over each dimension of a 2D array and -adds 1 to each item:: +adds 1 to each item: - def add_one(int[:,:] buf): - for x in xrange(buf.shape[0]): - for y in xrange(buf.shape[1]): - buf[x,y] += 1 +.. literalinclude:: ../../examples/userguide/memoryviews/add_one.pyx Indexing and slicing can be done with or without the GIL. It basically works like NumPy. If indices are specified for every dimension you will get an element of the base type (e.g. `int`). Otherwise, you will get a new view. An Ellipsis -means you get consecutive slices for every unspecified dimension:: - - cdef int[:, :, :] my_view = exporting_object +means you get consecutive slices for every unspecified dimension: - # These are all equivalent - my_view[10] - my_view[10, :, :] - my_view[10, ...] +.. literalinclude:: ../../examples/userguide/memoryviews/slicing.pyx Copying ------- -Memory views can be copied in place:: +Memory views can be copied in place: - cdef int[:, :, :] to_view, from_view - ... - - # copy the elements in from_view to to_view - to_view[...] = from_view - # or - to_view[:] = from_view - # or - to_view[:, :, :] = from_view +.. literalinclude:: ../../examples/userguide/memoryviews/copy.pyx They can also be copied with the ``copy()`` and ``copy_fortran()`` methods; see :ref:`view_copy_c_fortran`. @@ -199,10 +129,9 @@ ----------- In most cases (see below), the memoryview can be transposed in the same way that -NumPy slices can be transposed:: +NumPy slices can be transposed: - cdef int[:, ::1] c_contig = ... - cdef int[::1, :] f_contig = c_contig.T +.. literalinclude:: ../../examples/userguide/memoryviews/transpose.pyx This gives a new, transposed, view on the data. @@ -224,9 +153,43 @@ # 2D array with shape (50, 1) myslice[:, None] + # 3D array with shape (1, 10, 1) + myslice[None, 10:-20:2, None] + One may mix new axis indexing with all other forms of indexing and slicing. See also an example_. + +.. _readonly_views: + +Read-only views +--------------- + +Since Cython 0.28, the memoryview item type can be declared as ``const`` to +support read-only buffers as input: + +.. literalinclude:: ../../examples/userguide/memoryviews/np_flag_const.pyx + +Using a non-const memoryview with a binary Python string produces a runtime error. +You can solve this issue with a ``const`` memoryview: + +.. literalinclude:: ../../examples/userguide/memoryviews/view_string.pyx + +Note that this does not *require* the input buffer to be read-only:: + + a = np.linspace(0, 10, num=50) + myslice = a # read-only view of a writable buffer + +Writable buffers are still accepted by ``const`` views, but read-only +buffers are not accepted for non-const, writable views:: + + cdef double[:] myslice # a normal read/write memory view + + a = np.linspace(0, 10, num=50) + a.setflags(write=False) + myslice = a # ERROR: requesting writable memory view from read-only buffer! + + Comparison to the old buffer support ==================================== @@ -275,6 +238,8 @@ data array to themselves be pointers; Cython memoryviews do not yet support this. +.. _`new style buffers`: https://docs.python.org/3/c-api/buffer.html + .. _view_memory_layout: Memory layout @@ -325,7 +290,7 @@ Out[6]: (12, 4, 1) A Fortran contiguous array has the opposite memory ordering, with the elements -on the first axis closest togther in memory:: +on the first axis closest together in memory:: In [7]: f_contig = np.array(c_contig, order='F') In [8]: np.all(f_contig == c_contig) @@ -369,6 +334,8 @@ int [:, :, :] my_memoryview = obj +.. _c_and_fortran_contiguous_memoryviews: + C and Fortran contiguous memoryviews ------------------------------------ @@ -398,10 +365,10 @@ you will get a ``ValueError`` at runtime:: /Users/mb312/dev_trees/minimal-cython/mincy.pyx in init mincy (mincy.c:17267)() - 69 + 69 70 # But this isn't ---> 71 c_contiguous = np.array(c_contig, order='F') - 72 + 72 73 # Show the sum of all the arrays before altering it /Users/mb312/dev_trees/minimal-cython/stringsource in View.MemoryView.memoryview_cwrapper (mincy.c:9995)() @@ -460,31 +427,21 @@ * contiguous - contiguous and direct * indirect_contiguous - the list of pointers is contiguous -and they can be used like this:: - - from cython cimport view +and they can be used like this: - # direct access in both dimensions, strided in the first dimension, contiguous in the last - cdef int[:, ::view.contiguous] a +.. literalinclude:: ../../examples/userguide/memoryviews/memory_layout.pyx - # contiguous list of pointers to contiguous lists of ints - cdef int[::view.indirect_contiguous, ::1] b +Only the first, last or the dimension following an indirect dimension may be +specified contiguous: - # direct or indirect in the first dimension, direct in the second dimension - # strided in both dimensions - cdef int[::view.generic, :] c +.. literalinclude:: ../../examples/userguide/memoryviews/memory_layout_2.pyx -Only the first, last or the dimension following an indirect dimension may be -specified contiguous:: +:: # INVALID - cdef int[::view.contiguous, ::view.indirect, :] a - cdef int[::1, ::view.indirect, :] b + cdef int[::view.contiguous, ::view.indirect, :] d + cdef int[::1, ::view.indirect, :] e - # VALID - cdef int[::view.indirect, ::1, :] a - cdef int[::view.indirect, :, ::1] b - cdef int[::view.indirect_contiguous, ::1, :] The difference between the `contiguous` flag and the `::1` specifier is that the former specifies contiguity for only one dimension, whereas the latter specifies @@ -664,8 +621,61 @@ Unlike object attributes of extension classes, memoryview slices are not initialized to None. -.. _GIL: http://docs.python.org/dev/glossary.html#term-global-interpreter-lock -.. _new style buffers: http://docs.python.org/c-api/buffer.html -.. _pep 3118: http://www.python.org/peps/pep-3118.html -.. _NumPy: http://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html#memory-layout -.. _example: http://www.scipy.org/Numpy_Example_List#newaxis + +Pass data from a C function via pointer +======================================= + +Since use of pointers in C is ubiquitous, here we give a quick example of how +to call C functions whose arguments contain pointers. Let's suppose you want to +manage an array (allocate and deallocate) with NumPy (it can also be Python arrays, or +anything that supports the buffer interface), but you want to perform computation on this +array with an external C function implemented in :file:`C_func_file.c`: + +.. literalinclude:: ../../examples/userguide/memoryviews/C_func_file.c + :linenos: + +This file comes with a header file called :file:`C_func_file.h` containing: + +.. literalinclude:: ../../examples/userguide/memoryviews/C_func_file.h + :linenos: + +where ``arr`` points to the array and ``n`` is its size. + +You can call the function in a Cython file in the following way: + +.. literalinclude:: ../../examples/userguide/memoryviews/memview_to_c.pyx + :linenos: + +Several things to note: + - ``::1`` requests a C contiguous view, and fails if the buffer is not C contiguous. + See :ref:`c_and_fortran_contiguous_memoryviews`. + - ``&arr_memview[0]`` can be understood as 'the address of the first element of the + memoryview'. For contiguous arrays, this is equivalent to the + start address of the flat memory buffer. + - ``arr_memview.shape[0]`` could have been replaced by ``arr_memview.size``, + ``arr.shape[0]`` or ``arr.size``. But ``arr_memview.shape[0]`` is more efficient + because it doesn't require any Python interaction. + - ``multiply_by_10`` will perform computation in-place if the array passed is contiguous, + and will return a new numpy array if ``arr`` is not contiguous. + - If you are using Python arrays instead of numpy arrays, you don't need to check + if the data is stored contiguously as this is always the case. See :ref:`array-array`. + +This way, you can call the C function similar to a normal Python function, +and leave all the memory management and cleanup to NumPy arrays and Python's +object handling. For the details of how to compile and +call functions in C files, see :ref:`using_c_libraries`. + + +Performance: Disabling initialization checks +============================================ + +Every time the memoryview is accessed, Cython adds a check to make sure that it has been initialized. + +If you are looking for performance, you can disable them by setting the +``initializedcheck`` directive to ``False``. +See: :ref:`compiler-directives` for more information about this directive. + + +.. _GIL: https://docs.python.org/dev/glossary.html#term-global-interpreter-lock +.. _NumPy: https://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html#memory-layout +.. _example: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/migrating_to_cy30.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/migrating_to_cy30.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/migrating_to_cy30.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/migrating_to_cy30.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,174 @@ +.. highlight:: cython + +.. _cython30: + +********************************* +Migrating from Cython 0.29 to 3.0 +********************************* + +Cython 3.0 is a major revision of the compiler and the language +that comes with some backwards incompatible changes. +This document lists the important ones and explains how to deal with +them in existing code. + + +Python 3 syntax/semantics +========================= + +Cython 3.0 now uses Python 3 syntax and semantics by default, which previously +required setting the ``language_level`` `directive ` to +either ``3`` or ``3str``. +The new default setting is now ``language_level=3str``, which means Python 3 +semantics, but unprefixed strings are ``str`` objects, i.e. unicode text strings +under Python 3 and byte strings under Python 2.7. + +You can revert your code to the previous (Python 2.x) semantics by setting +``language_level=2``. + +Further semantic changes due to the language level include: + +* ``/``-division uses the true (float) division operator, unless ``cdivision`` is enabled. +* ``print`` is a function, not a statement. +* Python classes that are defined without bases (``class C: ...``) are "new-style" + classes also in Py2.x (if you never heard about "old-style classes", you're probably + happy without them). +* Annotations (type hints) are now stored as strings. + (`PEP 563 `_) +* ``StopIteration`` handling in generators has been changed according to + `PEP 479 `_. + + +Python semantics +================ + +Some Python compatibility bugs were fixed, e.g. + +* Subscripting (``x[1]``) now tries the mapping protocol before the sequence protocol. + (https://github.com/cython/cython/issues/1807) +* Exponentiation of integer literals now follows Python semantics and not C semantics. + (https://github.com/cython/cython/issues/2133) + + +Binding functions +================= + +The :ref:`binding directive ` is now enabled by default. +This makes Cython compiled Python (``def``) functions mostly compatible +with normal (non-compiled) Python functions, regarding signature introspection, +annotations, etc. + +It also makes them bind as methods in Python classes on attribute assignments, +thus the name. +If this is not intended, i.e. if a function is really meant to be a function +and never a method, you can disable the binding (and all other Python function +features) by setting ``binding=False`` or selectively adding a decorator +``@cython.binding(False)``. +In pure Python mode, the decorator was not available in Cython 0.29.16 yet, +but compiled code does not suffer from this. + +We recommend, however, to keep the new function features and instead deal +with the binding issue using the standard Python ``staticmethod()`` builtin. + +:: + + def func(self, b): ... + + class MyClass(object): + binding_method = func + + no_method = staticmethod(func) + + +Namespace packages +================== + +Cython now has support for loading pxd files also from namespace packages +according to `PEP-420 `_. +This might have an impact on the import path. + + +NumPy C-API +=========== + +Cython used to generate code that depended on the deprecated pre-NumPy-1.7 C-API. +This is no longer the case with Cython 3.0. + +You can now define the macro ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` +to get rid of the long-standing build warnings that the compiled C module +uses a deprecated API. Either per file:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or by setting it in your Extensions in ``setup.py``:: + + Extension(... + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")] + ) + +One side-effect of the different C-API usage is that your code may now +require a call to the `NumPy C-API initialisation function +`_ +where it previously got away without doing so. + +In order to reduce the user impact here, Cython 3.0 will now call it +automatically when it sees ``numpy`` being cimported, but the function +not being used. +In the (hopefully rare) cases where this gets in the way, the internal +C-API initialisation can be disabled by faking the use of the function +without actually calling it, e.g. + +:: + + # Explicitly disable the automatic initialisation of NumPy's C-API. + import_array + +Class-private name mangling +=========================== + +Cython has been updated to follow the `Python rules for class-private names +`_ +more closely. Essentially any name that starts with and doesn't end with +``__`` within a class is mangled with the class name. Most user code +should be unaffected -- unlike in Python unmangled global names will +still be matched to ensure it is possible to access C names +beginning with ``__``:: + + cdef extern void __foo() + + class C: # or "cdef class" + def call_foo(self): + return __foo() # still calls the global name + +What will no-longer work is overriding methods starting with ``__`` in +a ``cdef class``:: + + cdef class Base: + cdef __bar(self): + return 1 + + def call_bar(self): + return self.__bar() + + cdef class Derived(Base): + cdef __bar(self): + return 2 + +Here ``Base.__bar`` is mangled to ``_Base__bar`` and ``Derived.__bar`` +to ``_Derived__bar``. Therefore ``call_bar`` will always call +``_Base__bar``. This matches established Python behaviour and applies +for ``def``, ``cdef`` and ``cpdef`` methods and attributes. + +Arithmetic special methods +========================== + +The behaviour of arithmetic special methods (for example ``__add__`` +and ``__pow__``) of cdef classes has changed in Cython 3.0. They now +support separate "reversed" versions of these methods (e.g. +``__radd__``, ``__rpow__``) that behave like in pure Python. +The main incompatible change is that the type of the first operand +(usually ``__self__``) is now assumed to be that of the defining class, +rather than relying on the user to test and cast the type of each operand. + +The old behaviour can be restored with the +:ref:`directive ` ``c_api_binop_methods=True``. +More details are given in :ref:`arithmetic_methods`. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/numpy_pythran.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/numpy_pythran.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/numpy_pythran.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/numpy_pythran.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,55 @@ +.. highlight:: python + +.. _numpy-pythran: + +************************** +Pythran as a Numpy backend +************************** + +Using the flag ``--np-pythran``, it is possible to use the `Pythran`_ numpy +implementation for numpy related operations. One advantage to use this backend +is that the Pythran implementation uses C++ expression templates to save memory +transfers and can benefit from SIMD instructions of modern CPU. + +This can lead to really interesting speedup in some cases, going from 2 up to +16, depending on the targeted CPU architecture and the original algorithm. + +Please note that this feature is experimental. + +Usage example with setuptools +----------------------------- + +You first need to install Pythran. See its `documentation +`_ for more information. + +Then, simply add a ``cython: np_pythran=True`` directive at the top of the +Python files that needs to be compiled using Pythran numpy support. + +Here is an example of a simple ``setup.py`` file using setuptools: + +.. code:: + + from setuptools import setup + from Cython.Build import cythonize + + setup( + name = "My hello app", + ext_modules = cythonize('hello_pythran.pyx') + ) + +Then, with the following header in ``hello_pythran.pyx``: + +.. code:: + + # cython: np_pythran=True + +``hello_pythran.pyx`` will be compiled using Pythran numpy support. + +Please note that Pythran can further be tweaked by adding settings in the +``$HOME/.pythranrc`` file. For instance, this can be used to enable `Boost.SIMD`_ support. +See the `Pythran user manual +`_ for +more information. + +.. _Pythran: https://github.com/serge-sans-paille/pythran +.. _Boost.SIMD: https://github.com/NumScale/boost.simd diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/numpy_tutorial.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/numpy_tutorial.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/numpy_tutorial.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/numpy_tutorial.rst 2022-03-24 10:16:46.000000000 +0000 @@ -8,36 +8,30 @@ This tutorial is aimed at NumPy users who have no experience with Cython at all. If you have some knowledge of Cython you may want to skip to the -''Efficient indexing'' section which explains the new improvements made in -summer 2008. +''Efficient indexing'' section. The main scenario considered is NumPy end-use rather than NumPy/SciPy development. The reason is that Cython is not (yet) able to support functions -that are generic with respect to datatype and the number of dimensions in a +that are generic with respect to the number of dimensions in a high-level fashion. This restriction is much more severe for SciPy development than more specific, "end-user" functions. See the last section for more information on this. The style of this tutorial will not fit everybody, so you can also consider: -* Robert Bradshaw's `slides on cython for SciPy2008 - `_ - (a higher-level and quicker introduction) -* Basic Cython documentation (see `Cython front page `_). -* ``[:enhancements/buffer:Spec for the efficient indexing]`` - -.. Note:: - The fast array access documented below is a completely new feature, and - there may be bugs waiting to be discovered. It might be a good idea to do - a manual sanity check on the C code Cython generates before using this for - serious purposes, at least until some months have passed. +* Kurt Smith's `video tutorial of Cython at SciPy 2015 + `_. + The slides and notebooks of this talk are `on github + `_. +* Basic Cython documentation (see `Cython front page + `_). Cython at a glance -==================== +================== Cython is a compiler which compiles Python-like code files to C code. Still, ''Cython is not a Python to C translator''. That is, it doesn't take your full -program and "turns it into C" -- rather, the result makes full use of the +program and "turn it into C" -- rather, the result makes full use of the Python runtime environment. A way of looking at it may be that your code is still Python in that it runs within the Python runtime environment, but rather than compiling to interpreted Python bytecode one compiles to native machine @@ -51,12 +45,12 @@ of C libraries. When writing code in Cython you can call into C code as easily as into Python code. -Some Python constructs are not yet supported, though making Cython compile all -Python code is a stated goal (among the more important omissions are inner -functions and generator functions). +Very few Python constructs are not yet supported, though making Cython compile all +Python code is a stated goal, you can see the differences with Python in +:ref:`limitations `. Your Cython environment -======================== +======================= Using Cython consists of these steps: @@ -67,19 +61,25 @@ However there are several options to automate these steps: -1. The `SAGE `_ mathematics software system provides +1. The `SAGE `_ mathematics software system provides excellent support for using Cython and NumPy from an interactive command - line (like IPython) or through a notebook interface (like + line or through a notebook interface (like Maple/Mathematica). See `this documentation - `_. -2. A version of `pyximport `_ is shipped - with Cython, so that you can import pyx-files dynamically into Python and + `_. +2. Cython can be used as an extension within a Jupyter notebook, + making it easy to compile and use Cython code with just a ``%%cython`` + at the top of a cell. For more information see + :ref:`Using the Jupyter Notebook `. +3. A version of pyximport is shipped with Cython, + so that you can import pyx-files dynamically into Python and have them compiled automatically (See :ref:`pyximport`). -3. Cython supports distutils so that you can very easily create build scripts - which automate the process, this is the preferred method for full programs. -4. Manual compilation (see below) +4. Cython supports setuptools so that you can very easily create build scripts + which automate the process, this is the preferred method for + Cython implemented libraries and packages. + See :ref:`Basic setup.py `. +5. Manual compilation (see below) -.. Note:: +.. Note:: If using another interactive command line environment than SAGE, like IPython or Python itself, it is important that you restart the process when you recompile the module. It is not enough to issue an "import" @@ -88,16 +88,20 @@ Installation ============= -Unless you are used to some other automatic method: -`download Cython `_ (0.9.8.1.1 or later), unpack it, -and run the usual ```python setup.py install``. This will install a -``cython`` executable on your system. It is also possible to use Cython from -the source directory without installing (simply launch :file:`cython.py` in the -root directory). +If you already have a C compiler, just do: + +.. code-block:: bash + + pip install Cython + +otherwise, see :ref:`the installation page `. + As of this writing SAGE comes with an older release of Cython than required for this tutorial. So if using SAGE you should download the newest Cython and -then execute :: +then execute : + +.. code-block:: bash $ cd path/to/cython-distro $ path-to-sage/sage -python setup.py install @@ -108,7 +112,9 @@ ==================== As it is always important to know what is going on, I'll describe the manual -method here. First Cython is run:: +method here. First Cython is run: + +.. code-block:: bash $ cython yourmod.pyx @@ -120,13 +126,19 @@ Then we compile the C file. This may vary according to your system, but the C file should be built like Python was built. Python documentation for writing extensions should have some details. On Linux this often means something -like:: +like: + +.. code-block:: bash $ gcc -shared -pthread -fPIC -fwrapv -O2 -Wall -fno-strict-aliasing -I/usr/include/python2.7 -o yourmod.so yourmod.c ``gcc`` should have access to the NumPy C header files so if they are not installed at :file:`/usr/include/numpy` or similar you may need to pass another -option for those. +option for those. You only need to provide the NumPy headers if you write:: + + cimport numpy + +in your Cython code. This creates :file:`yourmod.so` in the same directory, which is importable by Python by using a normal ``import yourmod`` statement. @@ -134,236 +146,167 @@ The first Cython program ========================== -The code below does 2D discrete convolution of an image with a filter (and I'm -sure you can do better!, let it serve for demonstration purposes). It is both -valid Python and valid Cython code. I'll refer to it as both -:file:`convolve_py.py` for the Python version and :file:`convolve1.pyx` for the -Cython version -- Cython uses ".pyx" as its file suffix. - -.. code-block:: python - - from __future__ import division - import numpy as np - def naive_convolve(f, g): - # f is an image and is indexed by (v, w) - # g is a filter kernel and is indexed by (s, t), - # it needs odd dimensions - # h is the output image and is indexed by (x, y), - # it is not cropped - if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: - raise ValueError("Only odd dimensions on filter supported") - # smid and tmid are number of pixels between the center pixel - # and the edge, ie for a 5x5 filter they will be 2. - # - # The output size is calculated by adding smid, tmid to each - # side of the dimensions of the input image. - vmax = f.shape[0] - wmax = f.shape[1] - smax = g.shape[0] - tmax = g.shape[1] - smid = smax // 2 - tmid = tmax // 2 - xmax = vmax + 2*smid - ymax = wmax + 2*tmid - # Allocate result image. - h = np.zeros([xmax, ymax], dtype=f.dtype) - # Do convolution - for x in range(xmax): - for y in range(ymax): - # Calculate pixel value for h at (x,y). Sum one component - # for each pixel (s, t) of the filter g. - s_from = max(smid - x, -smid) - s_to = min((xmax - x) - smid, smid + 1) - t_from = max(tmid - y, -tmid) - t_to = min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = x - smid + s - w = y - tmid + t - value += g[smid - s, tmid - t] * f[v, w] - h[x, y] = value - return h +You can easily execute the code of this tutorial by +downloading `the Jupyter notebook `_. + +The code below does the equivalent of this function in numpy:: + + def compute_np(array_1, array_2, a, b, c): + return np.clip(array_1, 2, 10) * a + array_2 * b + c + +We'll say that ``array_1`` and ``array_2`` are 2D NumPy arrays of integer type and +``a``, ``b`` and ``c`` are three Python integers. + +This function uses NumPy and is already really fast, so it might be a bit overkill +to do it again with Cython. This is for demonstration purposes. Nonetheless, we +will show that we achieve a better speed and memory efficiency than NumPy at the cost of more verbosity. -This should be compiled to produce :file:`yourmod.so` (for Linux systems). We +This code computes the function with the loops over the two dimensions being unrolled. +It is both valid Python and valid Cython code. I'll refer to it as both +:file:`compute_py.py` for the Python version and :file:`compute_cy.pyx` for the +Cython version -- Cython uses ``.pyx`` as its file suffix (but it can also compile +``.py`` files). + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_py.py + +This should be compiled to produce :file:`compute_cy.so` for Linux systems +(on Windows systems, this will be a ``.pyd`` file). We run a Python session to test both the Python version (imported from ``.py``-file) and the compiled Cython module. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [1]: import numpy as np - In [2]: import convolve_py - In [3]: convolve_py.naive_convolve(np.array([[1, 1, 1]], dtype=np.int), - ... np.array([[1],[2],[1]], dtype=np.int)) - Out [3]: - array([[1, 1, 1], - [2, 2, 2], - [1, 1, 1]]) - In [4]: import convolve1 - In [4]: convolve1.naive_convolve(np.array([[1, 1, 1]], dtype=np.int), - ... np.array([[1],[2],[1]], dtype=np.int)) - Out [4]: - array([[1, 1, 1], - [2, 2, 2], - [1, 1, 1]]) - In [11]: N = 100 - In [12]: f = np.arange(N*N, dtype=np.int).reshape((N,N)) - In [13]: g = np.arange(81, dtype=np.int).reshape((9, 9)) - In [19]: %timeit -n2 -r3 convolve_py.naive_convolve(f, g) - 2 loops, best of 3: 1.86 s per loop - In [20]: %timeit -n2 -r3 convolve1.naive_convolve(f, g) - 2 loops, best of 3: 1.41 s per loop + In [2]: array_1 = np.random.uniform(0, 1000, size=(3000, 2000)).astype(np.intc) + In [3]: array_2 = np.random.uniform(0, 1000, size=(3000, 2000)).astype(np.intc) + In [4]: a = 4 + In [5]: b = 3 + In [6]: c = 9 + In [7]: def compute_np(array_1, array_2, a, b, c): + ...: return np.clip(array_1, 2, 10) * a + array_2 * b + c + In [8]: %timeit compute_np(array_1, array_2, a, b, c) + 103 ms ± 4.16 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) + + In [9]: import compute_py + In [10]: compute_py.compute(array_1, array_2, a, b, c) + 1min 10s ± 844 ms per loop (mean ± std. dev. of 7 runs, 1 loop each) + + In [11]: import compute_cy + In [12]: compute_cy.compute(array_1, array_2, a, b, c) + 56.5 s ± 587 ms per loop (mean ± std. dev. of 7 runs, 1 loop each) There's not such a huge difference yet; because the C code still does exactly what the Python interpreter does (meaning, for instance, that a new object is -allocated for each number used). Look at the generated html file and see what -is needed for even the simplest statements you get the point quickly. We need +allocated for each number used). + +You can look at the Python interaction and the generated C +code by using ``-a`` when calling Cython from the command +line, ``%%cython -a`` when using a Jupyter Notebook, or by using +``cythonize('compute_cy.pyx', annotate=True)`` when using a ``setup.py``. +Look at the generated html file and see what +is needed for even the simplest statements. You get the point quickly. We need to give Cython more information; we need to add types. + Adding types ============= To add types we use custom Cython syntax, so we are now breaking Python source -compatibility. Here's :file:`convolve2.pyx`. *Read the comments!* :: +compatibility. Here's :file:`compute_typed.pyx`. *Read the comments!* + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_typed.pyx - from __future__ import division - import numpy as np - # "cimport" is used to import special compile-time information - # about the numpy module (this is stored in a file numpy.pxd which is - # currently part of the Cython distribution). - cimport numpy as np - # We now need to fix a datatype for our arrays. I've used the variable - # DTYPE for this, which is assigned to the usual NumPy runtime - # type info object. - DTYPE = np.int - # "ctypedef" assigns a corresponding compile-time type to DTYPE_t. For - # every type in the numpy module there's a corresponding compile-time - # type with a _t-suffix. - ctypedef np.int_t DTYPE_t - # The builtin min and max functions works with Python objects, and are - # so very slow. So we create our own. - # - "cdef" declares a function which has much less overhead than a normal - # def function (but it is not Python-callable) - # - "inline" is passed on to the C compiler which may inline the functions - # - The C type "int" is chosen as return type and argument types - # - Cython allows some newer Python constructs like "a if x else b", but - # the resulting C file compiles with Python 2.3 through to Python 3.0 beta. - cdef inline int int_max(int a, int b): return a if a >= b else b - cdef inline int int_min(int a, int b): return a if a <= b else b - # "def" can type its arguments but not have a return type. The type of the - # arguments for a "def" function is checked at run-time when entering the - # function. - # - # The arrays f, g and h is typed as "np.ndarray" instances. The only effect - # this has is to a) insert checks that the function arguments really are - # NumPy arrays, and b) make some attribute access like f.shape[0] much - # more efficient. (In this example this doesn't matter though.) - def naive_convolve(np.ndarray f, np.ndarray g): - if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: - raise ValueError("Only odd dimensions on filter supported") - assert f.dtype == DTYPE and g.dtype == DTYPE - # The "cdef" keyword is also used within functions to type variables. It - # can only be used at the top indentation level (there are non-trivial - # problems with allowing them in other places, though we'd love to see - # good and thought out proposals for it). - # - # For the indices, the "int" type is used. This corresponds to a C int, - # other C types (like "unsigned int") could have been used instead. - # Purists could use "Py_ssize_t" which is the proper Python type for - # array indices. - cdef int vmax = f.shape[0] - cdef int wmax = f.shape[1] - cdef int smax = g.shape[0] - cdef int tmax = g.shape[1] - cdef int smid = smax // 2 - cdef int tmid = tmax // 2 - cdef int xmax = vmax + 2*smid - cdef int ymax = wmax + 2*tmid - cdef np.ndarray h = np.zeros([xmax, ymax], dtype=DTYPE) - cdef int x, y, s, t, v, w - # It is very important to type ALL your variables. You do not get any - # warnings if not, only much slower code (they are implicitly typed as - # Python objects). - cdef int s_from, s_to, t_from, t_to - # For the value variable, we want to use the same data type as is - # stored in the array, so we use "DTYPE_t" as defined above. - # NB! An important side-effect of this is that if "value" overflows its - # datatype size, it will simply wrap around like in C, rather than raise - # an error like in Python. - cdef DTYPE_t value - for x in range(xmax): - for y in range(ymax): - s_from = int_max(smid - x, -smid) - s_to = int_min((xmax - x) - smid, smid + 1) - t_from = int_max(tmid - y, -tmid) - t_to = int_min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = x - smid + s - w = y - tmid + t - value += g[smid - s, tmid - t] * f[v, w] - h[x, y] = value - return h - -At this point, have a look at the generated C code for :file:`convolve1.pyx` and -:file:`convolve2.pyx`. Click on the lines to expand them and see corresponding C. -(Note that this code annotation is currently experimental and especially -"trailing" cleanup code for a block may stick to the last expression in the -block and make it look worse than it is -- use some common sense). +.. figure:: compute_typed_html.jpg -* .. literalinclude: convolve1.html -* .. literalinclude: convolve2.html +At this point, have a look at the generated C code for :file:`compute_cy.pyx` and +:file:`compute_typed.pyx`. Click on the lines to expand them and see corresponding C. -Especially have a look at the for loops: In :file:`convolve1.c`, these are ~20 lines -of C code to set up while in :file:`convolve2.c` a normal C for loop is used. +Especially have a look at the ``for-loops``: In :file:`compute_cy.c`, these are ~20 lines +of C code to set up while in :file:`compute_typed.c` a normal C for loop is used. After building this and continuing my (very informal) benchmarks, I get: -.. sourcecode:: ipython +.. code-block:: ipythonconsole - In [21]: import convolve2 - In [22]: %timeit -n2 -r3 convolve2.naive_convolve(f, g) - 2 loops, best of 3: 828 ms per loop + In [13]: %timeit compute_typed.compute(array_1, array_2, a, b, c) + 26.5 s ± 422 ms per loop (mean ± std. dev. of 7 runs, 1 loop each) -Efficient indexing -==================== +So adding types does make the code faster, but nowhere +near the speed of NumPy? + +What happened is that most of the time spend in this code is spent in the following lines, +and those lines are slower to execute than in pure Python:: + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result[x, y] = tmp + c + +So what made those line so much slower than in the pure Python version? + +``array_1`` and ``array_2`` are still NumPy arrays, so Python objects, and expect +Python integers as indexes. Here we pass C int values. So every time +Cython reaches this line, it has to convert all the C integers to Python +int objects. Since this line is called very often, it outweighs the speed +benefits of the pure C loops that were created from the ``range()`` earlier. -There's still a bottleneck killing performance, and that is the array lookups -and assignments. The ``[]``-operator still uses full Python operations -- +Furthermore, ``tmp * a + array_2[x, y] * b`` returns a Python integer +and ``tmp`` is a C integer, so Cython has to do type conversions again. +In the end those types conversions add up. And made our computation really +slow. But this problem can be solved easily by using memoryviews. + +Efficient indexing with memoryviews +=================================== + +There are still two bottlenecks that degrade the performance, and that is the array lookups +and assignments, as well as C/Python types conversion. +The ``[]``-operator still uses full Python operations -- what we would like to do instead is to access the data buffer directly at C speed. What we need to do then is to type the contents of the :obj:`ndarray` objects. -We do this with a special "buffer" syntax which must be told the datatype -(first argument) and number of dimensions ("ndim" keyword-only argument, if -not provided then one-dimensional is assumed). +We do this with a memoryview. There is :ref:`a page in the Cython documentation +` dedicated to it. -More information on this syntax [:enhancements/buffer:can be found here]. +In short, memoryviews are C structures that can hold a pointer to the data +of a NumPy array and all the necessary buffer metadata to provide efficient +and safe access: dimensions, strides, item size, item type information, etc... +They also support slices, so they work even if +the NumPy array isn't contiguous in memory. +They can be indexed by C integers, thus allowing fast access to the +NumPy array data. -Showing the changes needed to produce :file:`convolve3.pyx` only:: +Here is how to declare a memoryview of integers:: - ... - def naive_convolve(np.ndarray[DTYPE_t, ndim=2] f, np.ndarray[DTYPE_t, ndim=2] g): - ... - cdef np.ndarray[DTYPE_t, ndim=2] h = ... - -Usage: - -.. sourcecode:: ipython - - In [18]: import convolve3 - In [19]: %timeit -n3 -r100 convolve3.naive_convolve(f, g) - 3 loops, best of 100: 11.6 ms per loop + cdef int [:] foo # 1D memoryview + cdef int [:, :] foo # 2D memoryview + cdef int [:, :, :] foo # 3D memoryview + ... # You get the idea. + +No data is copied from the NumPy array to the memoryview in our example. +As the name implies, it is only a "view" of the memory. So we can use the +view ``result_view`` for efficient indexing and at the end return the real NumPy +array ``result`` that holds the data that we operated on. + +Here is how to use them in our code: + +:file:`compute_memview.pyx` + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_memview.pyx + +Let's see how much faster accessing is now. + +.. code-block:: ipythonconsole + + In [22]: %timeit compute_memview.compute(array_1, array_2, a, b, c) + 22.9 ms ± 197 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) Note the importance of this change. +We're now 3081 times faster than an interpreted version of Python and 4.5 times +faster than NumPy. -*Gotcha*: This efficient indexing only affects certain index operations, -namely those with exactly ``ndim`` number of typed integer indices. So if -``v`` for instance isn't typed, then the lookup ``f[v, w]`` isn't -optimized. On the other hand this means that you can continue using Python -objects for sophisticated dynamic slicing etc. just as when the array is not -typed. +Memoryviews can be used with slices too, or even +with Python arrays. Check out the :ref:`memoryview page ` to +see what they can do for you. Tuning indexing further ======================== @@ -373,74 +316,39 @@ 1. Bounds checking is performed. 2. Negative indices are checked for and handled correctly. The code above is explicitly coded so that it doesn't use negative indices, and it - (hopefully) always access within bounds. We can add a decorator to disable - bounds checking:: + (hopefully) always access within bounds. + +With decorators, we can deactivate those checks:: + + ... + cimport cython + @cython.boundscheck(False) # Deactivate bounds checking + @cython.wraparound(False) # Deactivate negative indexing. + def compute(int[:, :] array_1, int[:, :] array_2, int a, int b, int c): + ... - ... - cimport cython - @cython.boundscheck(False) # turn off bounds-checking for entire function - def naive_convolve(np.ndarray[DTYPE_t, ndim=2] f, np.ndarray[DTYPE_t, ndim=2] g): - ... - Now bounds checking is not performed (and, as a side-effect, if you ''do'' happen to access out of bounds you will in the best case crash your program and in the worst case corrupt data). It is possible to switch bounds-checking mode in many ways, see :ref:`compiler-directives` for more information. -Negative indices are dealt with by ensuring Cython that the indices will be -positive, by casting the variables to unsigned integer types (if you do have -negative values, then this casting will create a very large positive value -instead and you will attempt to access out-of-bounds values). Casting is done -with a special ``<>``-syntax. The code below is changed to use either -unsigned ints or casting as appropriate:: - - ... - cdef int s, t # changed - cdef unsigned int x, y, v, w # changed - cdef int s_from, s_to, t_from, t_to - cdef DTYPE_t value - for x in range(xmax): - for y in range(ymax): - s_from = max(smid - x, -smid) - s_to = min((xmax - x) - smid, smid + 1) - t_from = max(tmid - y, -tmid) - t_to = min((ymax - y) - tmid, tmid + 1) - value = 0 - for s in range(s_from, s_to): - for t in range(t_from, t_to): - v = (x - smid + s) # changed - w = (y - tmid + t) # changed - value += g[(smid - s), (tmid - t)] * f[v, w] # changed - h[x, y] = value - ... - -(In the next Cython release we will likely add a compiler directive or -argument to the ``np.ndarray[]``-type specifier to disable negative indexing -so that casting so much isn't necessary; feedback on this is welcome.) - -The function call overhead now starts to play a role, so we compare the latter -two examples with larger N: - -.. sourcecode:: ipython - - In [11]: %timeit -n3 -r100 convolve4.naive_convolve(f, g) - 3 loops, best of 100: 5.97 ms per loop - In [12]: N = 1000 - In [13]: f = np.arange(N*N, dtype=np.int).reshape((N,N)) - In [14]: g = np.arange(81, dtype=np.int).reshape((9, 9)) - In [17]: %timeit -n1 -r10 convolve3.naive_convolve(f, g) - 1 loops, best of 10: 1.16 s per loop - In [18]: %timeit -n1 -r10 convolve4.naive_convolve(f, g) - 1 loops, best of 10: 597 ms per loop -(Also this is a mixed benchmark as the result array is allocated within the -function call.) +.. code-block:: ipythonconsole + + In [23]: %timeit compute_index.compute(array_1, array_2, a, b, c) + 16.8 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) + +We're faster than the NumPy version (6.2x). NumPy is really well written, +but does not performs operation lazily, resulting in a lot +of intermediate copy operations in memory. Our version is +very memory efficient and cache friendly because we +can execute the operations in a single run over the data. .. Warning:: Speed comes with some cost. Especially it can be dangerous to set typed - objects (like ``f``, ``g`` and ``h`` in our sample code) to ``None``. + objects (like ``array_1``, ``array_2`` and ``result_view`` in our sample code) to ``None``. Setting such objects to ``None`` is entirely legal, but all you can do with them is check whether they are None. All other use (attribute lookup or indexing) can potentially segfault or corrupt data (rather than raising exceptions as @@ -449,48 +357,151 @@ The actual rules are a bit more complicated but the main message is clear: Do not use typed objects without knowing that they are not set to ``None``. +Declaring the NumPy arrays as contiguous +======================================== + +For extra speed gains, if you know that the NumPy arrays you are +providing are contiguous in memory, you can declare the +memoryview as contiguous. + +We give an example on an array that has 3 dimensions. +If you want to give Cython the information that the data is C-contiguous +you have to declare the memoryview like this:: + + cdef int [:,:,::1] a + +If you want to give Cython the information that the data is Fortran-contiguous +you have to declare the memoryview like this:: + + cdef int [::1, :, :] a + +If all this makes no sense to you, you can skip this part, declaring +arrays as contiguous constrains the usage of your functions as it rejects array slices as input. +If you still want to understand what contiguous arrays are +all about, you can see `this answer on StackOverflow +`_. + +For the sake of giving numbers, here are the speed gains that you should +get by declaring the memoryviews as contiguous: + +.. code-block:: ipythonconsole + + In [23]: %timeit compute_contiguous.compute(array_1, array_2, a, b, c) + 11.1 ms ± 30.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) + +We're now around nine times faster than the NumPy version, and 6300 times +faster than the pure Python version! + +Making the function cleaner +=========================== + +Declaring types can make your code quite verbose. If you don't mind +Cython inferring the C types of your variables, you can use +the ``infer_types=True`` compiler directive at the top of the file. +It will save you quite a bit of typing. + +Note that since type declarations must happen at the top indentation level, +Cython won't infer the type of variables declared for the first time +in other indentation levels. It would change too much the meaning of +our code. This is why, we must still declare manually the type of the +``tmp``, ``x`` and ``y`` variable. + +And actually, manually giving the type of the ``tmp`` variable will +be useful when using fused types. + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_infer_types.pyx + +We now do a speed test: + +.. code-block:: ipythonconsole + + In [24]: %timeit compute_infer_types.compute(array_1, array_2, a, b, c) + 11.5 ms ± 261 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) + +Lo and behold, the speed has not changed. + More generic code ================== -It would be possible to do:: +All those speed gains are nice, but adding types constrains our code. +At the moment, it would mean that our function can only work with +NumPy arrays with the ``np.intc`` type. Is it possible to make our +code work for multiple NumPy data types? + +Yes, with the help of a new feature called fused types. +You can learn more about it at :ref:`this section of the documentation +`. +It is similar to C++ 's templates. It generates multiple function declarations +at compile time, and then chooses the right one at run-time based on the +types of the arguments provided. By comparing types in if-conditions, it +is also possible to execute entirely different code paths depending +on the specific data type. + +In our example, since we don't have access anymore to the NumPy's dtype +of our input arrays, we use those ``if-else`` statements to +know what NumPy data type we should use for our output array. + +In this case, our function now works for ints, doubles and floats. + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_fused_types.pyx + +We can check that the output type is the right one:: + + >>> compute(array_1, array_2, a, b, c).dtype + dtype('int32') + >>> compute(array_1.astype(np.double), array_2.astype(np.double), a, b, c).dtype + dtype('float64') + +We now do a speed test: + +.. code-block:: ipythonconsole + + In [25]: %timeit compute_fused_types.compute(array_1, array_2, a, b, c) + 11.5 ms ± 258 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) + +More versions of the function are created at compile time. So it makes +sense that the speed doesn't change for executing this function with +integers as before. + +Using multiple threads +====================== + +Cython has support for OpenMP. It also has some nice wrappers around it, +like the function :func:`prange`. You can see more information about Cython and +parallelism in :ref:`parallel`. Since we do elementwise operations, we can easily +distribute the work among multiple threads. It's important not to forget to pass the +correct arguments to the compiler to enable OpenMP. When using the Jupyter notebook, +you should use the cell magic like this: + +.. code-block:: ipython + + %%cython --force + # distutils: extra_compile_args=-fopenmp + # distutils: extra_link_args=-fopenmp + +The GIL must be released (see :ref:`Releasing the GIL `), so this is why we +declare our :func:`clip` function ``nogil``. + +.. literalinclude:: ../../examples/userguide/numpy_tutorial/compute_prange.pyx + +We can have substantial speed gains for minimal effort: + +.. code-block:: ipythonconsole + + In [25]: %timeit compute_prange.compute(array_1, array_2, a, b, c) + 9.33 ms ± 412 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) - def naive_convolve(object[DTYPE_t, ndim=2] f, ...): +We're now 7558 times faster than the pure Python version and 11.1 times faster +than NumPy! -i.e. use :obj:`object` rather than :obj:`np.ndarray`. Under Python 3.0 this -can allow your algorithm to work with any libraries supporting the buffer -interface; and support for e.g. the Python Imaging Library may easily be added -if someone is interested also under Python 2.x. - -There is some speed penalty to this though (as one makes more assumptions -compile-time if the type is set to :obj:`np.ndarray`, specifically it is -assumed that the data is stored in pure strided mode and not in indirect -mode). - -[:enhancements/buffer:More information] - -The future -============ - -These are some points to consider for further development. All points listed -here has gone through a lot of thinking and planning already; still they may -or may not happen depending on available developer time and resources for -Cython. - -1. Support for efficient access to structs/records stored in arrays; currently - only primitive types are allowed. -2. Support for efficient access to complex floating point types in arrays. The - main obstacle here is getting support for efficient complex datatypes in - Cython. -3. Calling NumPy/SciPy functions currently has a Python call overhead; it - would be possible to take a short-cut from Cython directly to C. (This does - however require some isolated and incremental changes to those libraries; - mail the Cython mailing list for details). -4. Efficient code that is generic with respect to the number of dimensions. - This can probably be done today by calling the NumPy C multi-dimensional - iterator API directly; however it would be nice to have for-loops over - :func:`enumerate` and :func:`ndenumerate` on NumPy arrays create efficient - code. -5. A high-level construct for writing type-generic code, so that one can write - functions that work simultaneously with many datatypes. Note however that a - macro preprocessor language can help with doing this for now. +Where to go from here? +====================== +* If you want to learn how to make use of `BLAS `_ + or `LAPACK `_ with Cython, you can watch + `the presentation of Ian Henriksen at SciPy 2015 + `_. +* If you want to learn how to use Pythran as backend in Cython, you + can see how in :ref:`Pythran as a NumPy backend `. + Note that using Pythran only works with the + :ref:`old buffer syntax ` and not yet with memoryviews. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/parallelism.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/parallelism.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/parallelism.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/parallelism.rst 2022-03-24 10:16:46.000000000 +0000 @@ -21,9 +21,7 @@ This function can be used for parallel loops. OpenMP automatically starts a thread pool and distributes the work according to the schedule - used. ``step`` must not be 0. This function can only be used with the - GIL released. If ``nogil`` is true, the loop will be wrapped in a nogil - section. + used. Thread-locality and reductions are automatically inferred for variables. @@ -36,88 +34,99 @@ Variables assigned to in a parallel with block will be private and unusable after the block, as there is no concept of a sequentially last value. - The ``schedule`` is passed to OpenMP and can be one of the following: - static: - If a chunksize is provided, iterations are distributed to all - threads ahead of time in blocks of the given chunksize. If no - chunksize is given, the iteration space is divided into chunks that - are approximately equal in size, and at most one chunk is assigned - to each thread in advance. - - This is most appropriate when the scheduling overhead matters and - the problem can be cut down into equally sized chunks that are - known to have approximately the same runtime. - - dynamic: - The iterations are distributed to threads as they request them, - with a default chunk size of 1. - - This is suitable when the runtime of each chunk differs and is not - known in advance and therefore a larger number of smaller chunks - is used in order to keep all threads busy. - - guided: - As with dynamic scheduling, the iterations are distributed to - threads as they request them, but with decreasing chunk size. The - size of each chunk is proportional to the number of unassigned - iterations divided by the number of participating threads, - decreasing to 1 (or the chunksize if provided). - - This has an advantage over pure dynamic scheduling when it turns - out that the last chunks take more time than expected or are - otherwise being badly scheduled, so that most threads start running - idle while the last chunks are being worked on by only a smaller - number of threads. - - runtime: - The schedule and chunk size are taken from the runtime scheduling - variable, which can be set through the ``openmp.omp_set_schedule()`` - function call, or the OMP_SCHEDULE environment variable. Note that - this essentially disables any static compile time optimisations of - the scheduling code itself and may therefore show a slightly worse - performance than when the same scheduling policy is statically - configured at compile time. - -.. auto The decision regarding scheduling is delegated to the -.. compiler and/or runtime system. The programmer gives -.. the implementation the freedom to choose any possible -.. mapping of iterations to threads in the team. - - The default schedule is implementation defined. For more information consult - the OpenMP specification [#]_. - - The ``num_threads`` argument indicates how many threads the team should consist of. If not given, - OpenMP will decide how many threads to use. Typically this is the number of cores available on - the machine. However, this may be controlled through the ``omp_set_num_threads()`` function, or - through the ``OMP_NUM_THREADS`` environment variable. - - The ``chunksize`` argument indicates the chunksize to be used for dividing the iterations among threads. - This is only valid for ``static``, ``dynamic`` and ``guided`` scheduling, and is optional. Different chunksizes - may give substantially different performance results, depending on the schedule, the load balance it provides, - the scheduling overhead and the amount of false sharing (if any). - - Example with a reduction:: - - from cython.parallel import prange - - cdef int i - cdef int sum = 0 - - for i in prange(n, nogil=True): - sum += i - - print sum - - Example with a typed memoryview (e.g. a NumPy array):: + :param start: + The index indicating the start of the loop (same as the start argument in range). - from cython.parallel import prange + :param stop: + The index indicating when to stop the loop (same as the stop argument in range). - def func(double[:] x, double alpha): - cdef Py_ssize_t i + :param step: + An integer giving the step of the sequence (same as the step argument in range). + It must not be 0. + + :param nogil: + This function can only be used with the GIL released. + If ``nogil`` is true, the loop will be wrapped in a nogil section. + + :param schedule: + The ``schedule`` is passed to OpenMP and can be one of the following: + + static: + If a chunksize is provided, iterations are distributed to all + threads ahead of time in blocks of the given chunksize. If no + chunksize is given, the iteration space is divided into chunks that + are approximately equal in size, and at most one chunk is assigned + to each thread in advance. + + This is most appropriate when the scheduling overhead matters and + the problem can be cut down into equally sized chunks that are + known to have approximately the same runtime. + + dynamic: + The iterations are distributed to threads as they request them, + with a default chunk size of 1. + + This is suitable when the runtime of each chunk differs and is not + known in advance and therefore a larger number of smaller chunks + is used in order to keep all threads busy. + + guided: + As with dynamic scheduling, the iterations are distributed to + threads as they request them, but with decreasing chunk size. The + size of each chunk is proportional to the number of unassigned + iterations divided by the number of participating threads, + decreasing to 1 (or the chunksize if provided). + + This has an advantage over pure dynamic scheduling when it turns + out that the last chunks take more time than expected or are + otherwise being badly scheduled, so that most threads start running + idle while the last chunks are being worked on by only a smaller + number of threads. + + runtime: + The schedule and chunk size are taken from the runtime scheduling + variable, which can be set through the ``openmp.omp_set_schedule()`` + function call, or the OMP_SCHEDULE environment variable. Note that + this essentially disables any static compile time optimisations of + the scheduling code itself and may therefore show a slightly worse + performance than when the same scheduling policy is statically + configured at compile time. + The default schedule is implementation defined. For more information consult + the OpenMP specification [#]_. + + .. auto The decision regarding scheduling is delegated to the + .. compiler and/or runtime system. The programmer gives + .. the implementation the freedom to choose any possible + .. mapping of iterations to threads in the team. + + + + :param num_threads: + The ``num_threads`` argument indicates how many threads the team should consist of. If not given, + OpenMP will decide how many threads to use. Typically this is the number of cores available on + the machine. However, this may be controlled through the ``omp_set_num_threads()`` function, or + through the ``OMP_NUM_THREADS`` environment variable. + + :param chunksize: + The ``chunksize`` argument indicates the chunksize to be used for dividing the iterations among threads. + This is only valid for ``static``, ``dynamic`` and ``guided`` scheduling, and is optional. Different chunksizes + may give substantially different performance results, depending on the schedule, the load balance it provides, + the scheduling overhead and the amount of false sharing (if any). - for i in prange(x.shape[0]): - x[i] = alpha * x[i] +Example with a reduction: + +.. literalinclude:: ../../examples/userguide/parallelism/simple_sum.pyx + +Example with a :term:`typed memoryview` (e.g. a NumPy array):: + + from cython.parallel import prange + + def func(double[:] x, double alpha): + cdef Py_ssize_t i + + for i in prange(x.shape[0]): + x[i] = alpha * x[i] .. function:: parallel(num_threads=None) @@ -139,7 +148,7 @@ with nogil, parallel(): local_buf = malloc(sizeof(int) * size) - if local_buf == NULL: + if local_buf is NULL: abort() # populate our local buffer in a sequential loop @@ -165,25 +174,9 @@ ========= To actually use the OpenMP support, you need to tell the C or C++ compiler to -enable OpenMP. For gcc this can be done as follows in a setup.py:: +enable OpenMP. For gcc this can be done as follows in a setup.py: - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize - - ext_modules = [ - Extension( - "hello", - ["hello.pyx"], - extra_compile_args=['-fopenmp'], - extra_link_args=['-fopenmp'], - ) - ] - - setup( - name='hello-parallel-world', - ext_modules=cythonize(ext_modules), - ) +.. literalinclude:: ../../examples/userguide/parallelism/setup.py For Microsoft Visual C++ compiler, use ``'/openmp'`` instead of ``'-fopenmp'``. @@ -199,39 +192,19 @@ body is skipped after the first break, return or exception for any subsequent iteration in any thread. It is undefined which value shall be returned if multiple different values may be returned, as the iterations are in no -particular order:: +particular order: - from cython.parallel import prange - - cdef int func(Py_ssize_t n): - cdef Py_ssize_t i - - for i in prange(n, nogil=True): - if i == 8: - with gil: - raise Exception() - elif i == 4: - break - elif i == 2: - return i +.. literalinclude:: ../../examples/userguide/parallelism/breaking_loop.pyx In the example above it is undefined whether an exception shall be raised, whether it will simply break or whether it will return 2. Using OpenMP Functions ====================== -OpenMP functions can be used by cimporting ``openmp``:: - - from cython.parallel cimport parallel - cimport openmp - - cdef int num_threads +OpenMP functions can be used by cimporting ``openmp``: - openmp.omp_set_dynamic(1) - with nogil, parallel(): - num_threads = openmp.omp_get_num_threads() - ... +.. literalinclude:: ../../examples/userguide/parallelism/cimport_openmp.pyx .. rubric:: References -.. [#] http://www.openmp.org/mp-documents/spec30.pdf +.. [#] https://www.openmp.org/mp-documents/spec30.pdf diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/pypy.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/pypy.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/pypy.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/pypy.rst 2022-03-24 10:16:46.000000000 +0000 @@ -1,8 +1,8 @@ Porting Cython code to PyPy =========================== -Since version 0.17, Cython has basic support for cpyext, the layer in -`PyPy `_ that emulates CPython's C-API. This is +Cython has basic support for cpyext, the layer in +`PyPy `_ that emulates CPython's C-API. This is achieved by making the generated C code adapt at C compile time, so the generated code will compile in both CPython and PyPy unchanged. @@ -118,7 +118,7 @@ code that tries to acquire the GIL "just in case", because it might be called with or without the GIL, will not work as expected in PyPy. See `PyGILState_Ensure should not deadlock if GIL already held -`_. +`_. Efficiency diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/pyrex_differences.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/pyrex_differences.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/pyrex_differences.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/pyrex_differences.rst 2022-03-24 10:16:46.000000000 +0000 @@ -6,14 +6,14 @@ Differences between Cython and Pyrex ************************************** -.. warning:: - Both Cython and Pyrex are moving targets. It has come to the point - that an explicit list of all the differences between the two - projects would be laborious to list and track, but hopefully - this high-level list gives an idea of the differences that +.. warning:: + Both Cython and Pyrex are moving targets. It has come to the point + that an explicit list of all the differences between the two + projects would be laborious to list and track, but hopefully + this high-level list gives an idea of the differences that are present. It should be noted that both projects make an effort - at mutual compatibility, but Cython's goal is to be as close to - and complete as Python as reasonable. + at mutual compatibility, but Cython's goal is to be as close to + and complete as Python as reasonable. Python 3 Support @@ -21,10 +21,14 @@ Cython creates ``.c`` files that can be built and used with both Python 2.x and Python 3.x. In fact, compiling your module with -Cython may very well be the easiest way to port code to Python 3. -We are also working to make the compiler run in both Python 2.x and 3.x. +Cython may very well be an easy way to port code to Python 3. -Many Python 3 constructs are already supported by Cython. +Cython also supports various syntax additions that came with +Python 3.0 and later major Python releases. If they do not conflict +with existing Python 2.x syntax or semantics, they are usually just +accepted by the compiler. Everything else depends on the +compiler directive ``language_level=3`` +(see :ref:`compiler directives`). List/Set/Dict Comprehensions ---------------------------- @@ -67,15 +71,14 @@ takes exactly two positional parameters and has two required keyword parameters. - Conditional expressions "x if b else y" ========================================= Conditional expressions as described in -http://www.python.org/dev/peps/pep-0308/:: +https://www.python.org/dev/peps/pep-0308/:: X if C else Y - + Only one of ``X`` and ``Y`` is evaluated (depending on the value of C). @@ -89,10 +92,10 @@ cdef inline int something_fast(int a, int b): return a*a + b - + Note that class-level :keyword:`cdef` functions are handled via a virtual function table, so the compiler won't be able to inline them in almost all -cases. +cases. Assignment on declaration (e.g. "cdef int spam = 5") ====================================================== @@ -103,24 +106,24 @@ i = 2 j = 5 k = 7 - + Now, with cython, one can write:: cdef int i = 2, j = 5, k = 7 - + The expression on the right hand side can be arbitrarily complicated, e.g.:: cdef int n = python_call(foo(x,y), a + b + c) - 32 - + 'by' expression in for loop (e.g. "for i from 0 <= i < 10 by 2") ================================================================== - + :: for i from 0 <= i < 10 by 2: print i - + yields:: @@ -207,7 +210,7 @@ x.foo() # will check to see if overridden A.foo(x) # will call A's implementation whether overridden or not -See :ref:`early-binding-for-speed` for explanation and usage tips. +See :ref:`early-binding-for-speed` for explanation and usage tips. .. _automatic-range-conversion: @@ -216,9 +219,9 @@ This will convert statements of the form ``for i in range(...)`` to ``for i from ...`` when ``i`` is any cdef'd integer type, and the direction (i.e. sign -of step) can be determined. +of step) can be determined. -.. warning:: +.. warning:: This may change the semantics if the range causes assignment to ``i`` to overflow. Specifically, if this option is set, an error @@ -256,30 +259,17 @@ arguments may increase on subclassing, but the argument types and order must remain the same. There is a slight performance penalty in some cases when a cdef/cpdef function without any optional is overridden with one that does have -default argument values. +default argument values. -For example, one can have the ``.pxd`` file:: +For example, one can have the ``.pxd`` file: - cdef class A: - cdef foo(self) - cdef class B(A) - cdef foo(self, x=*) - cdef class C(B): - cpdef foo(self, x=*, int k=*) +.. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pxd -with corresponding ``.pyx`` file:: +with corresponding ``.pyx`` file: - cdef class A: - cdef foo(self): - print "A" - cdef class B(A) - cdef foo(self, x=None) - print "B", x - cdef class C(B): - cpdef foo(self, x=True, int k=3) - print "C", x, k +.. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx -.. note:: +.. note:: this also demonstrates how :keyword:`cpdef` functions can override :keyword:`cdef` functions. @@ -320,7 +310,7 @@ Rather than introducing a new keyword ``typecheck`` as explained in the `Pyrex docs -`_, +`_, Cython emits a (non-spoofable and faster) typecheck whenever :func:`isinstance` is used with an extension type as the second parameter. @@ -330,13 +320,13 @@ Cython supports several ``from __future__ import ...`` directives, namely ``absolute_import``, ``unicode_literals``, ``print_function`` and ``division``. -With statements are always enabled. +With statements are always enabled. Pure Python mode ================ -Cython has support for compiling ``.py`` files, and +Cython has support for compiling ``.py`` files, and accepting type annotations using decorators and other -valid Python syntax. This allows the same source to -be interpreted as straight Python, or compiled for +valid Python syntax. This allows the same source to +be interpreted as straight Python, or compiled for optimized results. See :ref:`pure-mode` for more details. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/sharing_declarations.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/sharing_declarations.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/sharing_declarations.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/sharing_declarations.rst 2022-03-24 10:16:46.000000000 +0000 @@ -24,9 +24,9 @@ A ``.pxd`` file that consists solely of extern declarations does not need to correspond to an actual ``.pyx`` file or Python module. This can make it a -convenient place to put common declarations, for example declarations of +convenient place to put common declarations, for example declarations of functions from an :ref:`external library ` that one -wants to use in several modules. +wants to use in several modules. What a Definition File contains @@ -40,9 +40,9 @@ * The definition part of an extension type (see below). It cannot contain the implementations of any C or Python functions, or any -Python class definitions, or any executable statements. It is needed when one -wants to access :keyword:`cdef` attributes and methods, or to inherit from -:keyword:`cdef` classes defined in this module. +Python class definitions, or any executable statements. It is needed when one +wants to access :keyword:`cdef` attributes and methods, or to inherit from +:keyword:`cdef` classes defined in this module. .. note:: @@ -57,9 +57,9 @@ An implementation file can contain any kind of Cython statement, although there are some restrictions on the implementation part of an extension type if the -corresponding definition file also defines that type (see below). +corresponding definition file also defines that type (see below). If one doesn't need to :keyword:`cimport` anything from this module, then this -is the only file one needs. +is the only file one needs. .. _cimport: @@ -79,30 +79,15 @@ Here is an example. :file:`dishes.pxd` is a definition file which exports a C data type. :file:`restaurant.pyx` is an implementation file which imports and uses it. - -:file:`dishes.pxd`:: - cdef enum otherstuff: - sausage, eggs, lettuce - - cdef struct spamdish: - int oz_of_spam - otherstuff filler - -:file:`restaurant.pyx`:: - - cimport dishes - from dishes cimport spamdish - - cdef void prepare(spamdish *d): - d.oz_of_spam = 42 - d.filler = dishes.sausage - - def serve(): - cdef spamdish d - prepare(&d) - print "%d oz spam, filler no. %d" % (d.oz_of_spam, d.filler) - +:file:`dishes.pxd`: + +.. literalinclude:: ../../examples/userguide/sharing_declarations/dishes.pxd + +:file:`restaurant.pyx`: + +.. literalinclude:: ../../examples/userguide/sharing_declarations/restaurant.pyx + It is important to understand that the :keyword:`cimport` statement can only be used to import C data types, C functions and variables, and extension types. It cannot be used to import any Python objects, and (with one @@ -113,9 +98,9 @@ The exception is that when you use :keyword:`cimport` to import an extension type, its type object is imported at run time and made available by the name under which you imported it. Using :keyword:`cimport` to import extension types is covered in more -detail below. +detail below. -If a ``.pxd`` file changes, any modules that :keyword:`cimport` from it may need to be +If a ``.pxd`` file changes, any modules that :keyword:`cimport` from it may need to be recompiled. The ``Cython.Build.cythonize`` utility can take care of this for you. @@ -137,7 +122,7 @@ processing the ``.pyx`` file. -Using cimport to resolve naming conflicts +Using cimport to resolve naming conflicts ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The :keyword:`cimport` mechanism provides a clean and simple way to solve the @@ -146,24 +131,20 @@ for an imaginary module, and :keyword:`cimport` that module. You can then refer to the C functions by qualifying them with the name of the module. Here's an example: - -:file:`c_lunch.pxd`:: - cdef extern from "lunch.h": - void eject_tomato(float) +:file:`c_lunch.pxd`: -:file:`lunch.pyx`:: +.. literalinclude:: ../../examples/userguide/sharing_declarations/c_lunch.pxd - cimport c_lunch +:file:`lunch.pyx`: - def eject_tomato(float speed): - c_lunch.eject_tomato(speed) +.. literalinclude:: ../../examples/userguide/sharing_declarations/lunch.pyx You don't need any :file:`c_lunch.pyx` file, because the only things defined in :file:`c_lunch.pxd` are extern C entities. There won't be any actual ``c_lunch`` module at run time, but that doesn't matter; the :file:`c_lunch.pxd` file has done its job of providing an additional namespace -at compile time. +at compile time. Sharing C Functions @@ -173,34 +154,26 @@ :keyword:`cimport` by putting headers for them in the ``.pxd`` file, for example: -:file:`volume.pxd`:: +:file:`volume.pxd`: - cdef float cube(float) +.. literalinclude:: ../../examples/userguide/sharing_declarations/volume.pxd -:file:`volume.pyx`:: +:file:`volume.pyx`: - cdef float cube(float x): - return x * x * x +.. literalinclude:: ../../examples/userguide/sharing_declarations/volume.pyx -:file:`spammery.pyx`:: +:file:`spammery.pyx`: - from volume cimport cube - - def menu(description, size): - print description, ":", cube(size), \ - "cubic metres of spam" - - menu("Entree", 1) - menu("Main course", 3) - menu("Dessert", 2) +.. literalinclude:: ../../examples/userguide/sharing_declarations/spammery.pyx .. note:: When a module exports a C function in this way, an object appears in the module dictionary under the function's name. However, you can't make use of this object from Python, nor can you use it from Cython using a normal import - statement; you have to use :keyword:`cimport`. + statement; you have to use :keyword:`cimport`. +.. _sharing_extension_types: Sharing Extension Types ======================= @@ -220,38 +193,23 @@ Here is an example of a module which defines and exports an extension type, and another module which uses it: -:file:`Shrubbing.pxd`:: - - cdef class Shrubbery: - cdef int width - cdef int length +:file:`shrubbing.pxd`: -:file:`Shrubbing.pyx`:: +.. literalinclude:: ../../examples/userguide/sharing_declarations/shrubbing.pxd - cdef class Shrubbery: - def __cinit__(self, int w, int l): - self.width = w - self.length = l +:file:`shrubbing.pyx`: - def standard_shrubbery(): - return Shrubbery(3, 7) +.. literalinclude:: ../../examples/userguide/sharing_declarations/shrubbing.pyx -:file:`Landscaping.pyx`:: +:file:`landscaping.pyx`: - cimport Shrubbing - import Shrubbing - - cdef Shrubbing.Shrubbery sh - sh = Shrubbing.standard_shrubbery() - print "Shrubbery size is %d x %d" % (sh.width, sh.length) +.. literalinclude:: ../../examples/userguide/sharing_declarations/landscaping.pyx One would then need to compile both of these modules, e.g. using -:file:`setup.py`:: +:file:`setup.py`: - from distutils.core import setup - from Cython.Build import cythonize - setup(ext_modules = cythonize(["Landscaping.pyx", "Shrubbing.pyx"])) +.. literalinclude:: ../../examples/userguide/sharing_declarations/setup.py Some things to note about this example: @@ -263,3 +221,29 @@ doesn't bind the name Shrubbing in Landscaping's module namespace at run time, so to access :func:`Shrubbing.standard_shrubbery` we also need to ``import Shrubbing``. +* One caveat if you use setuptools instead of distutils, the default + action when running ``python setup.py install`` is to create a zipped + ``egg`` file which will not work with ``cimport`` for ``pxd`` files + when you try to use them from a dependent package. + To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. + +.. _versioning: + +Versioning +========== + +``.pxd`` files can be labelled with a minimum Cython version as part of +their file name, similar to the version tagging of ``.so`` files in PEP 3149. +For example a file called :file:`Shrubbing.cython-30.pxd` will only be +found by ``cimport Shrubbing`` on Cython 3.0 and higher. Cython will use the +file tagged with the highest compatible version number. + +Note that versioned files that are distributed across different directories +will not be found. Only the first directory in the Python module search +path in which a matching ``.pxd`` file is found will be considered. + +The purpose of this feature is to allow third-party packages to release +Cython interfaces to their packages that take advantage of the latest Cython +features while not breaking compatibility for users with older versions of Cython. +Users intending to use ``.pxd`` files solely within their own project +need not produce these tagged files. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/source_files_and_compilation.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/source_files_and_compilation.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/source_files_and_compilation.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/source_files_and_compilation.rst 2022-03-24 10:16:46.000000000 +0000 @@ -6,17 +6,45 @@ Source Files and Compilation **************************** -.. note:: See :ref:`compilation-reference` reference section for more details - Cython source file names consist of the name of the module followed by a ``.pyx`` extension, for example a module called primes would have a source file named :file:`primes.pyx`. +Cython code, unlike Python, must be compiled. This happens in two stages: + + * A ``.pyx`` file is compiled by Cython to a ``.c`` file. + + * The ``.c`` file is compiled by a C compiler to a ``.so`` file (or a + ``.pyd`` file on Windows) + Once you have written your ``.pyx`` file, there are a couple of ways of turning it -into an extension module. One way is to compile it manually with the Cython +into an extension module. + +The following sub-sections describe several ways to build your +extension modules, and how to pass directives to the Cython compiler. + +.. _compiling_command_line: + +Compiling from the command line +=============================== + +There are two ways of compiling from the command line. + +* The ``cython`` command takes a ``.py`` or ``.pyx`` file and + compiles it into a C/C++ file. + +* The ``cythonize`` command takes a ``.py`` or ``.pyx`` file and + compiles it into a C/C++ file. It then compiles the C/C++ file into + an extension module which is directly importable from Python. + + +Compiling with the ``cython`` command +------------------------------------- + +One way is to compile it manually with the Cython compiler, e.g.: -.. sourcecode:: text +.. code-block:: text $ cython primes.pyx @@ -25,35 +53,277 @@ platform for generating an extension module. For these options look at the official Python documentation. -The other, and probably better, way is to use the :mod:`distutils` extension +The other, and probably better, way is to use the :mod:`setuptools` extension provided with Cython. The benefit of this method is that it will give the platform specific compilation options, acting like a stripped down autotools. +Compiling with the ``cythonize`` command +---------------------------------------- + +Run the ``cythonize`` compiler command with your options and list of +``.pyx`` files to generate an extension module. For example: + +.. code-block:: bash + + $ cythonize -a -i yourmod.pyx + +This creates a ``yourmod.c`` file (or ``yourmod.cpp`` in C++ mode), compiles it, +and puts the resulting extension module (``.so`` or ``.pyd``, depending on your +platform) next to the source file for direct import (``-i`` builds "in place"). +The ``-a`` switch additionally produces an annotated html file of the source code. + +The ``cythonize`` command accepts multiple source files and glob patterns like +``**/*.pyx`` as argument and also understands the common ``-j`` option for +running multiple parallel build jobs. When called without further options, it +will only translate the source files to ``.c`` or ``.cpp`` files. Pass the +``-h`` flag for a complete list of supported options. + +There simpler command line tool ``cython`` only invokes the source code translator. + +In the case of manual compilation, how to compile your ``.c`` files will vary +depending on your operating system and compiler. The Python documentation for +writing extension modules should have some details for your system. On a Linux +system, for example, it might look similar to this: + +.. code-block:: bash + + $ gcc -shared -pthread -fPIC -fwrapv -O2 -Wall -fno-strict-aliasing \ + -I/usr/include/python3.5 -o yourmod.so yourmod.c + +(``gcc`` will need to have paths to your included header files and paths +to libraries you want to link with.) + +After compilation, a ``yourmod.so`` (:file:`yourmod.pyd` for Windows) +file is written into the target directory +and your module, ``yourmod``, is available for you to import as with any other +Python module. Note that if you are not relying on ``cythonize`` or setuptools, +you will not automatically benefit from the platform specific file extension +that CPython generates for disambiguation, such as +``yourmod.cpython-35m-x86_64-linux-gnu.so`` on a regular 64bit Linux installation +of CPython 3.5. + +.. _basic_setup.py: + Basic setup.py =============== -The distutils extension provided with Cython allows you to pass ``.pyx`` files +The setuptools extension provided with Cython allows you to pass ``.pyx`` files directly to the ``Extension`` constructor in your setup file. If you have a single Cython file that you want to turn into a compiled extension, say with filename :file:`example.pyx` the associated :file:`setup.py` would be:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( ext_modules = cythonize("example.pyx") - ) + ) + +If your build depends directly on Cython in this way, +then you may also want to inform pip that :mod:`Cython` is required for +:file:`setup.py` to execute, following `PEP 518 +`, creating a :file:`pyproject.toml` +file containing, at least: + +.. code-block:: ini + + + [build-system] + requires = ["setuptools", "wheel", "Cython"] -To understand the :file:`setup.py` more fully look at the official -:mod:`distutils` documentation. To compile the extension for use in the -current directory use: +To understand the :file:`setup.py` more fully look at the official `setuptools +documentation`_. To compile the extension for use in the current directory use: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace +Configuring the C-Build +------------------------ + +If you have include files in non-standard places you can pass an +``include_path`` parameter to ``cythonize``:: + + from setuptools import setup + from Cython.Build import cythonize + + setup( + name="My hello app", + ext_modules=cythonize("src/*.pyx", include_path=[...]), + ) + +Often, Python packages that offer a C-level API provide a way to find +the necessary include files, e.g. for NumPy:: + + include_path = [numpy.get_include()] + +.. note:: + + Using memoryviews or importing NumPy with ``import numpy`` does not mean that + you have to add the path to NumPy include files. You need to add this path only + if you use ``cimport numpy``. + +Despite this, you may still get warnings like the following from the compiler, +because Cython is not disabling the usage of the old deprecated Numpy API:: + + .../include/numpy/npy_1_7_deprecated_api.h:15:2: warning: #warning "Using deprecated NumPy API, disable it by " "#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-Wcpp] + +In Cython 3.0, you can get rid of this warning by defining the C macro +``NPY_NO_DEPRECATED_API`` as ``NPY_1_7_API_VERSION`` +in your build, e.g.:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or (see below):: + + Extension( + ..., + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], + ) + +With older Cython releases, setting this macro will fail the C compilation, +because Cython generates code that uses this deprecated C-API. However, the +warning has no negative effects even in recent NumPy versions including 1.18.x. +You can ignore it until you (or your library's users) switch to a newer NumPy +version that removes this long deprecated API, in which case you also need to +use Cython 3.0 or later. Thus, the earlier you switch to Cython 3.0, the +better for your users. + +If you need to specify compiler options, libraries to link with or other +linker options you will need to create ``Extension`` instances manually +(note that glob syntax can still be used to specify multiple extensions +in one line):: + + from setuptools import Extension, setup + from Cython.Build import cythonize + + extensions = [ + Extension("primes", ["primes.pyx"], + include_dirs=[...], + libraries=[...], + library_dirs=[...]), + # Everything but primes.pyx is included here. + Extension("*", ["*.pyx"], + include_dirs=[...], + libraries=[...], + library_dirs=[...]), + ] + setup( + name="My hello app", + ext_modules=cythonize(extensions), + ) + +Note that when using setuptools, you should import it before Cython, otherwise, +both might disagree about the class to use here. + +Note also that if you use setuptools instead of :mod:`distutils`, the default +action when running ``python setup.py install`` is to create a zipped +``egg`` file which will not work with ``cimport`` for ``pxd`` files +when you try to use them from a dependent package. +To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. + +If your options are static (for example you do not need to call a tool like +``pkg-config`` to determine them) you can also provide them directly in your +.pyx or .pxd source file using a special comment block at the start of the file:: + + # distutils: libraries = spam eggs + # distutils: include_dirs = /opt/food/include + +If you cimport multiple .pxd files defining libraries, then Cython +merges the list of libraries, so this works as expected (similarly +with other options, like ``include_dirs`` above). + +If you have some C files that have been wrapped with Cython and you want to +compile them into your extension, you can define the setuptools ``sources`` +parameter:: + + # distutils: sources = helper.c, another_helper.c + +Note that these sources are added to the list of sources of the current +extension module. Spelling this out in the :file:`setup.py` file looks +as follows:: + + from setuptools import Extension, setup + from Cython.Build import cythonize + + sourcefiles = ['example.pyx', 'helper.c', 'another_helper.c'] + + extensions = [Extension("example", sourcefiles)] + + setup( + ext_modules=cythonize(extensions) + ) + +The :class:`Extension` class takes many options, and a fuller explanation can +be found in the `setuptools documentation`_. Some useful options to know about +are ``include_dirs``, ``libraries``, and ``library_dirs`` which specify where +to find the ``.h`` and library files when linking to external libraries. + +.. _setuptools documentation: https://setuptools.readthedocs.io/ + +Sometimes this is not enough and you need finer customization of the +setuptools :class:`Extension`. +To do this, you can provide a custom function ``create_extension`` +to create the final :class:`Extension` object after Cython has processed +the sources, dependencies and ``# distutils`` directives but before the +file is actually Cythonized. +This function takes 2 arguments ``template`` and ``kwds``, where +``template`` is the :class:`Extension` object given as input to Cython +and ``kwds`` is a :class:`dict` with all keywords which should be used +to create the :class:`Extension`. +The function ``create_extension`` must return a 2-tuple +``(extension, metadata)``, where ``extension`` is the created +:class:`Extension` and ``metadata`` is metadata which will be written +as JSON at the top of the generated C files. This metadata is only used +for debugging purposes, so you can put whatever you want in there +(as long as it can be converted to JSON). +The default function (defined in ``Cython.Build.Dependencies``) is:: + + def default_create_extension(template, kwds): + if 'depends' in kwds: + include_dirs = kwds.get('include_dirs', []) + ["."] + depends = resolve_depends(kwds['depends'], include_dirs) + kwds['depends'] = sorted(set(depends + template.depends)) + + t = template.__class__ + ext = t(**kwds) + metadata = dict(distutils=kwds, module_name=kwds['name']) + return ext, metadata + +In case that you pass a string instead of an :class:`Extension` to +``cythonize()``, the ``template`` will be an :class:`Extension` without +sources. For example, if you do ``cythonize("*.pyx")``, +the ``template`` will be ``Extension(name="*.pyx", sources=[])``. + +Just as an example, this adds ``mylib`` as library to every extension:: + + from Cython.Build.Dependencies import default_create_extension + + def my_create_extension(template, kwds): + libs = kwds.get('libraries', []) + ["mylib"] + kwds['libraries'] = libs + return default_create_extension(template, kwds) + + ext_modules = cythonize(..., create_extension=my_create_extension) + +.. note:: + + If you Cythonize in parallel (using the ``nthreads`` argument), + then the argument to ``create_extension`` must be pickleable. + In particular, it cannot be a lambda function. + +.. _cythonize_arguments: + +Cythonize arguments +------------------- + +The function :func:`cythonize` can take extra arguments which will allow you to +customize your build. + +.. autofunction:: Cython.Build.cythonize + Multiple Cython Files in a Package =================================== @@ -75,88 +345,727 @@ ) +.. _distributing_cython_modules: + +Distributing Cython modules +---------------------------- + +It is strongly recommended that you distribute the generated ``.c`` files as well +as your Cython sources, so that users can install your module without needing +to have Cython available. + +It is also recommended that Cython compilation not be enabled by default in the +version you distribute. Even if the user has Cython installed, he/she probably +doesn't want to use it just to install your module. Also, the installed version +may not be the same one you used, and may not compile your sources correctly. + +This simply means that the :file:`setup.py` file that you ship with will just +be a normal setuptools file on the generated `.c` files, for the basic example +we would have instead:: + + from setuptools import Extension, setup + + setup( + ext_modules = [Extension("example", ["example.c"])] + ) + +This is easy to combine with :func:`cythonize` by changing the file extension +of the extension module sources:: + + from setuptools import Extension, setup + + USE_CYTHON = ... # command line option, try-import, ... + + ext = '.pyx' if USE_CYTHON else '.c' + + extensions = [Extension("example", ["example"+ext])] + + if USE_CYTHON: + from Cython.Build import cythonize + extensions = cythonize(extensions) + + setup( + ext_modules = extensions + ) + +If you have many extensions and want to avoid the additional complexity in the +declarations, you can declare them with their normal Cython sources and then +call the following function instead of ``cythonize()`` to adapt the sources +list in the Extensions when not using Cython:: + + import os.path + + def no_cythonize(extensions, **_ignore): + for extension in extensions: + sources = [] + for sfile in extension.sources: + path, ext = os.path.splitext(sfile) + if ext in ('.pyx', '.py'): + if extension.language == 'c++': + ext = '.cpp' + else: + ext = '.c' + sfile = path + ext + sources.append(sfile) + extension.sources[:] = sources + return extensions + +Another option is to make Cython a setup dependency of your system and use +Cython's build_ext module which runs ``cythonize`` as part of the build process:: + + setup( + extensions = [Extension("*", ["*.pyx"])], + cmdclass={'build_ext': Cython.Build.build_ext}, + ... + ) + +This depends on pip knowing that :mod:`Cython` is a setup dependency, by having +a :file:`pyproject.toml` file:: + + [build-system] + requires = ["setuptools", "wheel", "Cython"] + +If you want to expose the C-level interface of your library for other +libraries to cimport from, use package_data to install the ``.pxd`` files, +e.g.:: + + setup( + package_data = { + 'my_package': ['*.pxd'], + 'my_package/sub_package': ['*.pxd'], + }, + ... + ) + +These ``.pxd`` files need not have corresponding ``.pyx`` +modules if they contain purely declarations of external libraries. + +Remember that if you use setuptools instead of distutils, the default +action when running ``python setup.py install`` is to create a zipped +``egg`` file which will not work with ``cimport`` for ``pxd`` files +when you try to use them from a dependent package. +To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. + + +.. _integrating_multiple_modules: + +Integrating multiple modules +============================ + +In some scenarios, it can be useful to link multiple Cython modules +(or other extension modules) into a single binary, e.g. when embedding +Python in another application. This can be done through the inittab +import mechanism of CPython. + +Create a new C file to integrate the extension modules and add this +macro to it:: + + #if PY_MAJOR_VERSION < 3 + # define MODINIT(name) init ## name + #else + # define MODINIT(name) PyInit_ ## name + #endif + +If you are only targeting Python 3.x, just use ``PyInit_`` as prefix. + +Then, for each of the modules, declare its module init function +as follows, replacing ``some_module_name`` with the name of the module:: + + PyMODINIT_FUNC MODINIT(some_module_name) (void); + +In C++, declare them as ``extern C``. + +If you are not sure of the name of the module init function, refer +to your generated module source file and look for a function name +starting with ``PyInit_``. + +Next, before you start the Python runtime from your application code +with ``Py_Initialize()``, you need to initialise the modules at runtime +using the ``PyImport_AppendInittab()`` C-API function, again inserting +the name of each of the modules:: + + PyImport_AppendInittab("some_module_name", MODINIT(some_module_name)); + +This enables normal imports for the embedded extension modules. + +In order to prevent the joined binary from exporting all of the module +init functions as public symbols, Cython 0.28 and later can hide these +symbols if the macro ``CYTHON_NO_PYINIT_EXPORT`` is defined while +C-compiling the module C files. + +Also take a look at the `cython_freeze +`_ tool. +It can generate the necessary boilerplate code for linking one or more +modules into a single Python executable. + + .. _pyximport: -Pyximport -=========== +Compiling with :mod:`pyximport` +=============================== + +For building Cython modules during development without explicitly +running ``setup.py`` after each change, you can use :mod:`pyximport`:: -.. TODO add some text about how this is Paul Prescods code. Also change the - tone to be more universal (i.e. remove all the I statements) + >>> import pyximport; pyximport.install() + >>> import helloworld + Hello World -Cython is a compiler. Therefore it is natural that people tend to go -through an edit/compile/test cycle with Cython modules. But my personal -opinion is that one of the deep insights in Python's implementation is -that a language can be compiled (Python modules are compiled to ``.pyc`` -files) and hide that compilation process from the end-user so that they -do not have to worry about it. Pyximport does this for Cython modules. -For instance if you write a Cython module called :file:`foo.pyx`, with -Pyximport you can import it in a regular Python module like this:: +This allows you to automatically run Cython on every ``.pyx`` that +Python is trying to import. You should use this for simple Cython +builds only where no extra C libraries and no special building setup +is needed. +It is also possible to compile new ``.py`` modules that are being +imported (including the standard library and installed packages). For +using this feature, just tell that to :mod:`pyximport`:: - import pyximport; pyximport.install() - import foo + >>> pyximport.install(pyimport=True) -Doing so will result in the compilation of :file:`foo.pyx` (with appropriate -exceptions if it has an error in it). +In the case that Cython fails to compile a Python module, :mod:`pyximport` +will fall back to loading the source modules instead. -If you would always like to import Cython files without building them -specially, you can also the first line above to your :file:`sitecustomize.py`. -That will install the hook every time you run Python. Then you can use -Cython modules just with simple import statements. I like to test my -Cython modules like this: +Note that it is not recommended to let :mod:`pyximport` build code +on end user side as it hooks into their import system. The best way +to cater for end users is to provide pre-built binary packages in the +`wheel `_ packaging format. -.. sourcecode:: text - $ python -c "import foo" +Arguments +--------- + +The function ``pyximport.install()`` can take several arguments to +influence the compilation of Cython or Python files. + +.. autofunction:: pyximport.install + Dependency Handling -------------------- -In Pyximport 1.1 it is possible to declare that your module depends on -multiple files, (likely ``.h`` and ``.pxd`` files). If your Cython module is -named ``foo`` and thus has the filename :file:`foo.pyx` then you should make -another file in the same directory called :file:`foo.pyxdep`. The -:file:`modname.pyxdep` file can be a list of filenames or "globs" (like -``*.pxd`` or ``include/*.h``). Each filename or glob must be on a separate -line. Pyximport will check the file date for each of those files before -deciding whether to rebuild the module. In order to keep track of the -fact that the dependency has been handled, Pyximport updates the -modification time of your ".pyx" source file. Future versions may do -something more sophisticated like informing distutils of the -dependencies directly. +Since :mod:`pyximport` does not use :func:`cythonize()` internally, it currently +requires a different setup for dependencies. It is possible to declare that +your module depends on multiple files, (likely ``.h`` and ``.pxd`` files). +If your Cython module is named ``foo`` and thus has the filename +:file:`foo.pyx` then you should create another file in the same directory +called :file:`foo.pyxdep`. The :file:`modname.pyxdep` file can be a list of +filenames or "globs" (like ``*.pxd`` or ``include/*.h``). Each filename or +glob must be on a separate line. Pyximport will check the file date for each +of those files before deciding whether to rebuild the module. In order to +keep track of the fact that the dependency has been handled, Pyximport updates +the modification time of your ".pyx" source file. Future versions may do +something more sophisticated like informing setuptools of the dependencies +directly. + Limitations ------------ +:mod:`pyximport` does not use :func:`cythonize()`. Thus it is not +possible to do things like using compiler directives at +the top of Cython files or compiling Cython code to C++. + Pyximport does not give you any control over how your Cython file is -compiled. Usually the defaults are fine. You might run into problems if +compiled. Usually the defaults are fine. You might run into problems if you wanted to write your program in half-C, half-Cython and build them -into a single library. Pyximport 1.2 will probably do this. +into a single library. + +Pyximport does not hide the setuptools/GCC warnings and errors generated +by the import process. Arguably this will give you better feedback if +something went wrong and why. And if nothing went wrong it will give you +the warm fuzzy feeling that pyximport really did rebuild your module as it +was supposed to. + +Basic module reloading support is available with the option ``reload_support=True``. +Note that this will generate a new module filename for each build and thus +end up loading multiple shared libraries into memory over time. CPython has limited +support for reloading shared libraries as such, +see `PEP 489 `_. + +Pyximport puts both your ``.c`` file and the platform-specific binary into +a separate build directory, usually ``$HOME/.pyxblx/``. To copy it back +into the package hierarchy (usually next to the source file) for manual +reuse, you can pass the option ``inplace=True``. + + +.. _compiling_with_cython_inline: + +Compiling with ``cython.inline`` +================================= + +One can also compile Cython in a fashion similar to SciPy's ``weave.inline``. +For example:: + + >>> import cython + >>> def f(a): + ... ret = cython.inline("return a+b", b=3) + ... + +Unbound variables are automatically pulled from the surrounding local +and global scopes, and the result of the compilation is cached for +efficient re-use. + +Compiling with ``cython.compile`` +================================= + +Cython supports transparent compiling of the cython code in a function using the +``@cython.compile`` dedorator:: + + @cython.compile + def plus(a, b): + return a + b + +Parameters of the decorated function cannot have type declarations. Their types are +automatically determined from values passed to the function, thus leading to one or more +specialised compiled functions for the respective argument types. +Executing example:: + + import cython + + @cython.compile + def plus(a, b): + return a + b + + print(plus('3', '5')) + print(plus(3, 5)) + +will produce following output:: + + 35 + 8 + +.. _compiling_with_sage: -Pyximport does not hide the Distutils/GCC warnings and errors generated -by the import process. Arguably this will give you better feedback if -something went wrong and why. And if nothing went wrong it will give you -the warm fuzzy that pyximport really did rebuild your module as it was -supposed to. - -For further thought and discussion ------------------------------------- - -I don't think that Python's :func:`reload` will do anything for changed -``.so``'s on some (all?) platforms. It would require some (easy) -experimentation that I haven't gotten around to. But reload is rarely used in -applications outside of the Python interactive interpreter and certainly not -used much for C extension modules. Info about Windows -``_ - -``setup.py install`` does not modify :file:`sitecustomize.py` for you. Should it? -Modifying Python's "standard interpreter" behaviour may be more than -most people expect of a package they install.. - -Pyximport puts your ``.c`` file beside your ``.pyx`` file (analogous to -``.pyc`` beside ``.py``). But it puts the platform-specific binary in a -build directory as per normal for Distutils. If I could wave a magic -wand and get Cython or distutils or whoever to put the build directory I -might do it but not necessarily: having it at the top level is *VERY* -*HELPFUL* for debugging Cython problems. +Compiling with Sage +=================== + +The Sage notebook allows transparently editing and compiling Cython +code simply by typing ``%cython`` at the top of a cell and evaluate +it. Variables and functions defined in a Cython cell are imported into the +running session. Please check `Sage documentation +`_ for details. + +You can tailor the behavior of the Cython compiler by specifying the +directives below. + +.. _compiling_notebook: + +Compiling with a Jupyter Notebook +================================= + +It's possible to compile code in a notebook cell with Cython. +For this you need to load the Cython magic:: + + %load_ext cython + +Then you can define a Cython cell by writing ``%%cython`` on top of it. +Like this:: + + %%cython + + cdef int a = 0 + for i in range(10): + a += i + print(a) + +Note that each cell will be compiled into a separate extension module. So if you use a package in a Cython +cell, you will have to import this package in the same cell. It's not enough to +have imported the package in a previous cell. Cython will tell you that there are +"undefined global names" at compilation time if you don't comply. + +The global names (top level functions, classes, variables and modules) of the +cell are then loaded into the global namespace of the notebook. So in the +end, it behaves as if you executed a Python cell. + +Additional allowable arguments to the Cython magic are listed below. +You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook. + +============================================ ======================================================================================================================================= + +-a, --annotate Produce a colorized HTML version of the source. + +--annotate-fullc Produce a colorized HTML version of the source which includes entire generated C/C++-code. + +-+, --cplus Output a C++ rather than C file. + +-f, --force Force the compilation of a new module, even if the source has been previously compiled. + +-3 Select Python 3 syntax + +-2 Select Python 2 syntax + +-c=COMPILE_ARGS, --compile-args=COMPILE_ARGS Extra flags to pass to compiler via the extra_compile_args. + +--link-args LINK_ARGS Extra flags to pass to linker via the extra_link_args. + +-l LIB, --lib LIB Add a library to link the extension against (can be specified multiple times). + +-L dir Add a path to the list of library directories (can be specified multiple times). + +-I INCLUDE, --include INCLUDE Add a path to the list of include directories (can be specified multiple times). + +-S, --src Add a path to the list of src files (can be specified multiple times). + +-n NAME, --name NAME Specify a name for the Cython module. + +--pgo Enable profile guided optimisation in the C compiler. Compiles the cell twice and executes it in between to generate a runtime profile. + +--verbose Print debug information like generated .c/.cpp file location and exact gcc/g++ command invoked. + +============================================ ======================================================================================================================================= + + +.. _compiler_options: + +Compiler options +---------------- + +Compiler options can be set in the :file:`setup.py`, before calling :func:`cythonize`, +like this:: + + from setuptools import setup + + from Cython.Build import cythonize + from Cython.Compiler import Options + + Options.docstrings = False + + setup( + name = "hello", + ext_modules = cythonize("lib.pyx"), + ) + +Here are the options that are available: + +.. autodata:: Cython.Compiler.Options.docstrings +.. autodata:: Cython.Compiler.Options.embed_pos_in_docstring +.. pre_import +.. autodata:: Cython.Compiler.Options.generate_cleanup_code +.. autodata:: Cython.Compiler.Options.clear_to_none +.. autodata:: Cython.Compiler.Options.annotate +.. annotate_coverage_xml +.. autodata:: Cython.Compiler.Options.fast_fail +.. autodata:: Cython.Compiler.Options.warning_errors +.. autodata:: Cython.Compiler.Options.error_on_unknown_names +.. autodata:: Cython.Compiler.Options.error_on_uninitialized +.. autodata:: Cython.Compiler.Options.convert_range +.. autodata:: Cython.Compiler.Options.cache_builtins +.. autodata:: Cython.Compiler.Options.gcc_branch_hints +.. autodata:: Cython.Compiler.Options.lookup_module_cpdef +.. autodata:: Cython.Compiler.Options.embed +.. old_style_globals +.. autodata:: Cython.Compiler.Options.cimport_from_pyx +.. autodata:: Cython.Compiler.Options.buffer_max_dims +.. autodata:: Cython.Compiler.Options.closure_freelist_size + + +.. _compiler-directives: + +Compiler directives +==================== + +Compiler directives are instructions which affect the behavior of +Cython code. Here is the list of currently supported directives: + +``binding`` (True / False) + Controls whether free functions behave more like Python's CFunctions + (e.g. :func:`len`) or, when set to True, more like Python's functions. + When enabled, functions will bind to an instance when looked up as a + class attribute (hence the name) and will emulate the attributes + of Python functions, including introspections like argument names and + annotations. + + Default is True. + + .. versionchanged:: 3.0.0 + Default changed from False to True + +``boundscheck`` (True / False) + If set to False, Cython is free to assume that indexing operations + ([]-operator) in the code will not cause any IndexErrors to be + raised. Lists, tuples, and strings are affected only if the index + can be determined to be non-negative (or if ``wraparound`` is False). + Conditions which would normally trigger an IndexError may instead cause + segfaults or data corruption if this is set to False. + Default is True. + +``wraparound`` (True / False) + In Python, arrays and sequences can be indexed relative to the end. + For example, A[-1] indexes the last value of a list. + In C, negative indexing is not supported. + If set to False, Cython is allowed to neither check for nor correctly + handle negative indices, possibly causing segfaults or data corruption. + If bounds checks are enabled (the default, see ``boundschecks`` above), + negative indexing will usually raise an ``IndexError`` for indices that + Cython evaluates itself. + However, these cases can be difficult to recognise in user code to + distinguish them from indexing or slicing that is evaluated by the + underlying Python array or sequence object and thus continues to support + wrap-around indices. + It is therefore safest to apply this option only to code that does not + process negative indices at all. + Default is True. + +``initializedcheck`` (True / False) + If set to True, Cython checks that + - a memoryview is initialized whenever its elements are accessed + or assigned to. + - a C++ class is initialized when it is accessed + (only when ``cpp_locals`` is on) + Setting this to False disables these checks. + Default is True. + +``nonecheck`` (True / False) + If set to False, Cython is free to assume that native field + accesses on variables typed as an extension type, or buffer + accesses on a buffer variable, never occurs when the variable is + set to ``None``. Otherwise a check is inserted and the + appropriate exception is raised. This is off by default for + performance reasons. Default is False. + +``overflowcheck`` (True / False) + If set to True, raise errors on overflowing C integer arithmetic + operations. Incurs a modest runtime penalty, but is much faster than + using Python ints. Default is False. + +``overflowcheck.fold`` (True / False) + If set to True, and overflowcheck is True, check the overflow bit for + nested, side-effect-free arithmetic expressions once rather than at every + step. Depending on the compiler, architecture, and optimization settings, + this may help or hurt performance. A simple suite of benchmarks can be + found in ``Demos/overflow_perf.pyx``. Default is True. + +``embedsignature`` (True / False) + If set to True, Cython will embed a textual copy of the call + signature in the docstring of all Python visible functions and + classes. Tools like IPython and epydoc can thus display the + signature, which cannot otherwise be retrieved after + compilation. Default is False. + +``cdivision`` (True / False) + If set to False, Cython will adjust the remainder and quotient + operators C types to match those of Python ints (which differ when + the operands have opposite signs) and raise a + ``ZeroDivisionError`` when the right operand is 0. This has up to + a 35% speed penalty. If set to True, no checks are performed. See + `CEP 516 `_. Default + is False. + +``cdivision_warnings`` (True / False) + If set to True, Cython will emit a runtime warning whenever + division is performed with negative operands. See `CEP 516 + `_. Default is + False. + +``always_allow_keywords`` (True / False) + When disabled, uses the ``METH_NOARGS`` and ``METH_O`` signatures when + constructing functions/methods which take zero or one arguments. Has no + effect on special methods and functions with more than one argument. The + ``METH_NOARGS`` and ``METH_O`` signatures provide slightly faster + calling conventions but disallow the use of keywords. + +``c_api_binop_methods`` (True / False) + When enabled, makes the special binary operator methods (``__add__``, etc.) + behave according to the low-level C-API slot semantics, i.e. only a single + method implements both the normal and reversed operator. This used to be + the default in Cython 0.x and was now replaced by Python semantics, i.e. the + default in Cython 3.x and later is ``False``. + +``profile`` (True / False) + Write hooks for Python profilers into the compiled C code. Default + is False. + +``linetrace`` (True / False) + Write line tracing hooks for Python profilers or coverage reporting + into the compiled C code. This also enables profiling. Default is + False. Note that the generated module will not actually use line + tracing, unless you additionally pass the C macro definition + ``CYTHON_TRACE=1`` to the C compiler (e.g. using the setuptools option + ``define_macros``). Define ``CYTHON_TRACE_NOGIL=1`` to also include + ``nogil`` functions and sections. + +``infer_types`` (True / False) + Infer types of untyped variables in function bodies. Default is + None, indicating that only safe (semantically-unchanging) inferences + are allowed. + In particular, inferring *integral* types for variables *used in arithmetic + expressions* is considered unsafe (due to possible overflow) and must be + explicitly requested. + +``language_level`` (2/3/3str) + Globally set the Python language level to be used for module + compilation. Default is compatibility with Python 2. To enable + Python 3 source code semantics, set this to 3 (or 3str) at the start + of a module or pass the "-3" or "--3str" command line options to the + compiler. The ``3str`` option enables Python 3 semantics but does + not change the ``str`` type and unprefixed string literals to + ``unicode`` when the compiled code runs in Python 2.x. + Note that cimported files inherit this setting from the module + being compiled, unless they explicitly set their own language level. + Included source files always inherit this setting. + +``c_string_type`` (bytes / str / unicode) + Globally set the type of an implicit coercion from char* or std::string. + +``c_string_encoding`` (ascii, default, utf-8, etc.) + Globally set the encoding to use when implicitly coercing char* or std:string + to a unicode object. Coercion from a unicode object to C type is only allowed + when set to ``ascii`` or ``default``, the latter being utf-8 in Python 3 and + nearly-always ascii in Python 2. + +``type_version_tag`` (True / False) + Enables the attribute cache for extension types in CPython by setting the + type flag ``Py_TPFLAGS_HAVE_VERSION_TAG``. Default is True, meaning that + the cache is enabled for Cython implemented types. To disable it + explicitly in the rare cases where a type needs to juggle with its ``tp_dict`` + internally without paying attention to cache consistency, this option can + be set to False. + +``unraisable_tracebacks`` (True / False) + Whether to print tracebacks when suppressing unraisable exceptions. + +``iterable_coroutine`` (True / False) + `PEP 492 `_ specifies that async-def + coroutines must not be iterable, in order to prevent accidental misuse in + non-async contexts. However, this makes it difficult and inefficient to write + backwards compatible code that uses async-def coroutines in Cython but needs to + interact with async Python code that uses the older yield-from syntax, such as + asyncio before Python 3.5. This directive can be applied in modules or + selectively as decorator on an async-def coroutine to make the affected + coroutine(s) iterable and thus directly interoperable with yield-from. + +``annotation_typing`` (True / False) + Uses function argument annotations to determine the type of variables. Default + is True, but can be disabled. Since Python does not enforce types given in + annotations, setting to False gives greater compatibility with Python code. + Must be set globally. + +``emit_code_comments`` (True / False) + Copy the original source code line by line into C code comments in the generated + code file to help with understanding the output. + This is also required for coverage analysis. + +``cpp_locals`` (True / False) + Make C++ variables behave more like Python variables by allowing them to be + "unbound" instead of always default-constructing them at the start of a + function. See :ref:`cpp_locals directive` for more detail. + +.. _configurable_optimisations: + +Configurable optimisations +-------------------------- + +``optimize.use_switch`` (True / False) + Whether to expand chained if-else statements (including statements like + ``if x == 1 or x == 2:``) into C switch statements. This can have performance + benefits if there are lots of values but cause compiler errors if there are any + duplicate values (which may not be detectable at Cython compile time for all + C constants). Default is True. + +``optimize.unpack_method_calls`` (True / False) + Cython can generate code that optimistically checks for Python method objects + at call time and unpacks the underlying function to call it directly. This + can substantially speed up method calls, especially for builtins, but may also + have a slight negative performance impact in some cases where the guess goes + completely wrong. + Disabling this option can also reduce the code size. Default is True. + +.. _warnings: + +Warnings +-------- + +All warning directives take True / False as options +to turn the warning on / off. + +``warn.undeclared`` (default False) + Warns about any variables that are implicitly declared without a ``cdef`` declaration + +``warn.unreachable`` (default True) + Warns about code paths that are statically determined to be unreachable, e.g. + returning twice unconditionally. + +``warn.maybe_uninitialized`` (default False) + Warns about use of variables that are conditionally uninitialized. + +``warn.unused`` (default False) + Warns about unused variables and declarations + +``warn.unused_arg`` (default False) + Warns about unused function arguments + +``warn.unused_result`` (default False) + Warns about unused assignment to the same name, such as + ``r = 2; r = 1 + 2`` + +``warn.multiple_declarators`` (default True) + Warns about multiple variables declared on the same line with at least one pointer type. + For example ``cdef double* a, b`` - which, as in C, declares ``a`` as a pointer, ``b`` as + a value type, but could be mininterpreted as declaring two pointers. + + +.. _how_to_set_directives: + +How to set directives +--------------------- + +Globally +::::::::: + +One can set compiler directives through a special header comment near the top of the file, like this:: + + # cython: language_level=3, boundscheck=False + +The comment must appear before any code (but can appear after other +comments or whitespace). + +One can also pass a directive on the command line by using the -X switch: + +.. code-block:: bash + + $ cython -X boundscheck=True ... + +Directives passed on the command line will override directives set in +header comments. + +Locally +:::::::: + +For local blocks, you need to cimport the special builtin ``cython`` +module:: + + #!python + cimport cython + +Then you can use the directives either as decorators or in a with +statement, like this:: + + #!python + @cython.boundscheck(False) # turn off boundscheck for this function + def f(): + ... + # turn it temporarily on again for this block + with cython.boundscheck(True): + ... + +.. Warning:: These two methods of setting directives are **not** + affected by overriding the directive on the command-line using the + -X option. + +In :file:`setup.py` +::::::::::::::::::: + +Compiler directives can also be set in the :file:`setup.py` file by passing a keyword +argument to ``cythonize``:: + + from setuptools import setup + from Cython.Build import cythonize + + setup( + name="My hello app", + ext_modules=cythonize('hello.pyx', compiler_directives={'embedsignature': True}), + ) +This will override the default directives as specified in the ``compiler_directives`` dictionary. +Note that explicit per-file or local directives as explained above take precedence over the +values passed to ``cythonize``. diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/special_methods.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/special_methods.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/special_methods.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/special_methods.rst 2022-03-24 10:16:46.000000000 +0000 @@ -9,83 +9,119 @@ counterparts or have no direct Python counterparts, and require special mention. -.. Note: Everything said on this page applies only to extension types, defined - with the :keyword:`cdef class` statement. It doesn't apply to classes defined with the - Python :keyword:`class` statement, where the normal Python rules apply. - +.. Note:: + + Everything said on this page applies only to extension types, defined + with the :keyword:`cdef` class statement. It doesn't apply to classes defined with the + Python :keyword:`class` statement, where the normal Python rules apply. + +.. _declaration: + Declaration ------------ Special methods of extension types must be declared with :keyword:`def`, not :keyword:`cdef`. This does not impact their performance--Python uses different -calling conventions to invoke these special methods. +calling conventions to invoke these special methods. + +.. _docstrings: Docstrings ----------- Currently, docstrings are not fully supported in some special methods of extension types. You can place a docstring in the source to serve as a comment, but it -won't show up in the corresponding :attr:`__doc__` attribute at run time. (This -seems to be is a Python limitation -- there's nowhere in the `PyTypeObject` -data structure to put such docstrings.) +won't show up in the corresponding :attr:`__doc__` attribute at run time. (This +seems to be is a Python limitation -- there's nowhere in the `PyTypeObject` +data structure to put such docstrings.) + + +.. _initialisation_methods: Initialisation methods: :meth:`__cinit__` and :meth:`__init__` --------------------------------------------------------------- -There are two methods concerned with initialising the object. - -The :meth:`__cinit__` method is where you should perform basic C-level -initialisation of the object, including allocation of any C data structures -that your object will own. You need to be careful what you do in the -:meth:`__cinit__` method, because the object may not yet be fully valid Python -object when it is called. Therefore, you should be careful invoking any Python -operations which might touch the object; in particular, its methods. - -By the time your :meth:`__cinit__` method is called, memory has been allocated for the -object and any C attributes it has have been initialised to 0 or null. (Any -Python attributes have also been initialised to None, but you probably -shouldn't rely on that.) Your :meth:`__cinit__` method is guaranteed to be called -exactly once. - -If your extension type has a base type, the :meth:`__cinit__` method of the base type -is automatically called before your :meth:`__cinit__` method is called; you cannot -explicitly call the inherited :meth:`__cinit__` method. If you need to pass a modified -argument list to the base type, you will have to do the relevant part of the -initialisation in the :meth:`__init__` method instead (where the normal rules for -calling inherited methods apply). - -Any initialisation which cannot safely be done in the :meth:`__cinit__` method should -be done in the :meth:`__init__` method. By the time :meth:`__init__` is called, the object is -a fully valid Python object and all operations are safe. Under some -circumstances it is possible for :meth:`__init__` to be called more than once or not -to be called at all, so your other methods should be designed to be robust in -such situations. +There are two methods concerned with initialising the object, the normal Python +:meth:`__init__` method and a special :meth:`__cinit__` method where basic +C level initialisation can be performed. + +The main difference between the two is when they are called. +The :meth:`__cinit__` method is guaranteed to be called as part of the object +allocation, but before the object is fully initialised. Specifically, methods +and object attributes that belong to subclasses or that were overridden by +subclasses may not have been initialised at all yet and must not be used by +:meth:`__cinit__` in a base class. Note that the object allocation in Python +clears all fields and sets them to zero (or ``NULL``). Cython additionally +takes responsibility of setting all object attributes to ``None``, but again, +this may not yet have been done for the attributes defined or overridden by +subclasses. If your object needs anything more than this basic attribute +clearing in order to get into a correct and safe state, :meth:`__cinit__` +may be a good place to do it. + +The :meth:`__init__` method, on the other hand, works exactly like in Python. +It is called after allocation and basic initialisation of the object, including +the complete inheritance chain. +By the time :meth:`__init__` is called, the object is a fully valid Python object +and all operations are safe. Any initialisation which cannot safely be done in +the :meth:`__cinit__` method should be done in the :meth:`__init__` method. +However, as in Python, it is the responsibility of the subclasses to call up the +hierarchy and make sure that the :meth:`__init__` methods in the base class are +called correctly. If a subclass forgets (or refuses) to call the :meth:`__init__` +method of one of its base classes, that method will not be called. +Also, if the object gets created by calling directly its :meth:`__new__` method [#]_ +(as opposed to calling the class itself), then none of the :meth:`__init__` +methods will be called. + +The :meth:`__cinit__` method is where you should perform basic safety C-level +initialisation of the object, possibly including allocation of any C data +structures that your object will own. In contrast to :meth:`__init__`, +your :meth:`__cinit__` method is guaranteed to be called exactly once. + +If your extension type has a base type, any existing :meth:`__cinit__` methods in +the base type hierarchy are automatically called before your :meth:`__cinit__` +method. You cannot explicitly call the inherited :meth:`__cinit__` methods, and the +base types are free to choose whether they implement :meth:`__cinit__` at all. +If you need to pass a modified argument list to the base type, you will have to do +the relevant part of the initialisation in the :meth:`__init__` method instead, +where the normal rules for calling inherited methods apply. Any arguments passed to the constructor will be passed to both the -:meth:`__cinit__` method and the :meth:`__init__` method. If you anticipate -subclassing your extension type in Python, you may find it useful to give the -:meth:`__cinit__` method `*` and `**` arguments so that it can accept and -ignore extra arguments. Otherwise, any Python subclass which has an -:meth:`__init__` with a different signature will have to override -:meth:`__new__` [#]_ as well as :meth:`__init__`, which the writer of a Python -class wouldn't expect to have to do. Alternatively, as a convenience, if you declare -your :meth:`__cinit__`` method to take no arguments (other than self) it -will simply ignore any extra arguments passed to the constructor without -complaining about the signature mismatch. - -.. Note: Older Cython files may use :meth:`__new__` rather than :meth:`__cinit__`. The two are synonyms. - The name change from :meth:`__new__` to :meth:`__cinit__` was to avoid - confusion with Python :meth:`__new__` (which is an entirely different - concept) and eventually the use of :meth:`__new__` in Cython will be - disallowed to pave the way for supporting Python-style :meth:`__new__` +:meth:`__cinit__` method and the :meth:`__init__` method. If you anticipate +subclassing your extension type, you may find it useful to give the +:meth:`__cinit__` method ``*`` and ``**`` arguments so that it can accept and +ignore arbitrary extra arguments, since the arguments that are passed through +the hierarchy during allocation cannot be changed by subclasses. +Alternatively, as a convenience, if you declare your :meth:`__cinit__` method +to take no arguments (other than self) it will simply ignore any extra arguments +passed to the constructor without complaining about the signature mismatch. + +.. Note:: + + All constructor arguments will be passed as Python objects. + This implies that non-convertible C types such as pointers or C++ objects + cannot be passed into the constructor, neither from Python nor from Cython code. + If this is needed, use a factory function or method instead that handles the + object initialisation. + It often helps to directly call the :meth:`__new__` method in this function to + explicitly bypass the call to the :meth:`__init__` constructor. + + See :ref:`existing-pointers-instantiation` for an example. + +.. Note:: + + Implementing a :meth:`__cinit__` method currently excludes the type from + :ref:`auto-pickling `. + +.. [#] https://docs.python.org/reference/datamodel.html#object.__new__ + -.. [#] http://docs.python.org/reference/datamodel.html#object.__new__ +.. _finalization_method: -Finalization method: :meth:`__dealloc__` ----------------------------------------- +Finalization methods: :meth:`__dealloc__` and :meth:`__del__` +------------------------------------------------------------- The counterpart to the :meth:`__cinit__` method is the :meth:`__dealloc__` method, which should perform the inverse of the :meth:`__cinit__` method. Any -C data that you explicitly allocated (e.g. via malloc) in your -:meth:`__cinit__` method should be freed in your :meth:`__dealloc__` method. +C data that you explicitly allocated (e.g. via malloc) in your +:meth:`__cinit__` method should be freed in your :meth:`__dealloc__` method. You need to be careful what you do in a :meth:`__dealloc__` method. By the time your :meth:`__dealloc__` method is called, the object may already have been partially @@ -97,54 +133,114 @@ You don't need to worry about deallocating Python attributes of your object, because that will be done for you by Cython after your :meth:`__dealloc__` method -returns. +returns. When subclassing extension types, be aware that the :meth:`__dealloc__` method of the superclass will always be called, even if it is overridden. This is in contrast to typical Python behavior where superclass methods will not be executed unless they are explicitly called by the subclass. -.. Note:: There is no :meth:`__del__` method for extension types. +Python 3.4 made it possible for extension types to safely define +finalizers for objects. When running a Cython module on Python 3.4 and +higher you can add a :meth:`__del__` method to extension types in +order to perform Python cleanup operations. When the :meth:`__del__` +is called the object is still in a valid state (unlike in the case of +:meth:`__dealloc__`), permitting the use of Python operations +on its class members. On Python <3.4 :meth:`__del__` will not be called. + +.. _arithmetic_methods: Arithmetic methods ------------------- -Arithmetic operator methods, such as :meth:`__add__`, behave differently from their -Python counterparts. There are no separate "reversed" versions of these -methods (:meth:`__radd__`, etc.) Instead, if the first operand cannot perform the -operation, the same method of the second operand is called, with the operands -in the same order. - -This means that you can't rely on the first parameter of these methods being -"self" or being the right type, and you should test the types of both operands -before deciding what to do. If you can't handle the combination of types you've -been given, you should return `NotImplemented`. - -This also applies to the in-place arithmetic method :meth:`__ipow__`. It doesn't apply -to any of the other in-place methods (:meth:`__iadd__`, etc.) which always -take `self` as the first argument. +Arithmetic operator methods, such as :meth:`__add__`, used to behave differently +from their Python counterparts in Cython 0.x, following the low-level semantics +of the C-API slot functions. Since Cython 3.0, they are called in the same way +as in Python, including the separate "reversed" versions of these methods +(:meth:`__radd__`, etc.). + +Previously, if the first operand could not perform the operation, the same method +of the second operand was called, with the operands in the same order. +This means that you could not rely on the first parameter of these methods being +"self" or being the right type, and you needed to test the types of both operands +before deciding what to do. + +If backwards compatibility is needed, the normal operator method (``__add__``, etc.) +can still be implemented to support both variants, applying a type check to the +arguments. The reversed method (``__radd__``, etc.) can always be implemented +with ``self`` as first argument and will be ignored by older Cython versions, whereas +Cython 3.x and later will only call the normal method with the expected argument order, +and otherwise call the reversed method instead. + +Alternatively, the old Cython 0.x (or native C-API) behaviour is still available with +the directive ``c_api_binop_methods=True``. + +If you can't handle the combination of types you've been given, you should return +``NotImplemented``. This will let Python's operator implementation first try to apply +the reversed operator to the second operand, and failing that as well, report an +appropriate error to the user. + +This change in behaviour also applies to the in-place arithmetic method :meth:`__ipow__`. +It does not apply to any of the other in-place methods (:meth:`__iadd__`, etc.) +which always take ``self`` as the first argument. + +.. _rich_comparisons: Rich comparisons ----------------- -There are no separate methods for the individual rich comparison operations -(:meth:`__eq__`, :meth:`__le__`, etc.) Instead there is a single method -:meth:`__richcmp__` which takes an integer indicating which operation is to be -performed, as follows: - -+-----+-----+ -| < | 0 | -+-----+-----+ -| == | 2 | -+-----+-----+ -| > | 4 | -+-----+-----+ -| <= | 1 | -+-----+-----+ -| != | 3 | -+-----+-----+ -| >= | 5 | -+-----+-----+ +There are a few ways to implement comparison methods. +Depending on the application, one way or the other may be better: + +* Use the 6 Python + `special methods `_ + :meth:`__eq__`, :meth:`__lt__`, etc. + This is supported since Cython 0.27 and works exactly as in plain Python classes. + +* Use a single special method :meth:`__richcmp__`. + This implements all rich comparison operations in one method. + The signature is ``def __richcmp__(self, other, int op)``. + The integer argument ``op`` indicates which operation is to be performed + as shown in the table below: + + +-----+-------+ + | < | Py_LT | + +-----+-------+ + | == | Py_EQ | + +-----+-------+ + | > | Py_GT | + +-----+-------+ + | <= | Py_LE | + +-----+-------+ + | != | Py_NE | + +-----+-------+ + | >= | Py_GE | + +-----+-------+ + + These constants can be cimported from the ``cpython.object`` module. + +* Use the ``@cython.total_ordering`` decorator, which is a low-level + re-implementation of the `functools.total_ordering + `_ + decorator specifically for ``cdef`` classes. (Normal Python classes can use + the original ``functools`` decorator.) + + .. code-block:: cython + + @cython.total_ordering + cdef class ExtGe: + cdef int x + + def __ge__(self, other): + if not isinstance(other, ExtGe): + return NotImplemented + return self.x >= (other).x + + def __eq__(self, other): + return isinstance(other, ExtGe) and self.x == (other).x + + +.. _the__next__method: The :meth:`__next__` method ---------------------------- @@ -154,6 +250,8 @@ supply a next method which calls your :meth:`__next__`. Do *NOT* explicitly give your type a :meth:`next` method, or bad things could happen. +.. _special_methods_table: + Special Method Table --------------------- @@ -164,10 +262,12 @@ You don't have to declare your method as taking these parameter types. If you declare different types, conversions will be performed as necessary. - + General ^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#special-method-names + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ @@ -177,15 +277,13 @@ +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __dealloc__ |self | | Basic deallocation (no direct Python equivalent) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __cmp__ |x, y | int | 3-way comparison | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __richcmp__ |x, y, int op | object | Rich comparison (no direct Python equivalent) | +| __cmp__ |x, y | int | 3-way comparison (Python 2 only) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __str__ |self | object | str(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __repr__ |self | object | repr(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __hash__ |self | int | Hash function | +| __hash__ |self | Py_hash_t | Hash function (returns 32/64 bit integer) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __call__ |self, ... | object | self(...) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -200,54 +298,92 @@ | __delattr__ |self, name | | Delete attribute | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ +Rich comparison operators +^^^^^^^^^^^^^^^^^^^^^^^^^ + +https://docs.python.org/3/reference/datamodel.html#basic-customization + +You can choose to either implement the standard Python special methods +like :meth:`__eq__` or the single special method :meth:`__richcmp__`. +Depending on the application, one way or the other may be better. + ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| Name | Parameters | Return type | Description | ++=======================+=======================================+=============+========================================================+ +| __eq__ |self, y | object | self == y | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __ne__ |self, y | object | self != y (falls back to ``__eq__`` if not available) | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __lt__ |self, y | object | self < y | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __gt__ |self, y | object | self > y | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __le__ |self, y | object | self <= y | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __ge__ |self, y | object | self >= y | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ +| __richcmp__ |self, y, int op | object | Joined rich comparison method for all of the above | +| | | | (no direct Python equivalent) | ++-----------------------+---------------------------------------+-------------+--------------------------------------------------------+ + Arithmetic operators ^^^^^^^^^^^^^^^^^^^^ -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __add__ | x, y | object | binary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __sub__ | x, y | object | binary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mul__ | x, y | object | `*` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __div__ | x, y | object | `/` operator for old-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __floordiv__ | x, y | object | `//` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __truediv__ | x, y | object | `/` operator for new-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mod__ | x, y | object | `%` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __divmod__ | x, y | object | combined div and mod | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pow__ | x, y, z | object | `**` operator or pow(x, y, z) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __neg__ | self | object | unary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pos__ | self | object | unary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __abs__ | self | object | absolute value | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __nonzero__ | self | int | convert to boolean | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __invert__ | self | object | `~` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __lshift__ | x, y | object | `<<` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __rshift__ | x, y | object | `>>` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __and__ | x, y | object | `&` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __or__ | x, y | object | `|` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __xor__ | x, y | object | `^` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ +https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| Name | Parameters | Return type | Description | ++=============================+====================+=============+=====================================================+ +| __add__, __radd__ | self, other | object | binary `+` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __sub__, __rsub__ | self, other | object | binary `-` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __mul__, __rmul__ | self, other | object | `*` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __div__, __rdiv__ | self, other | object | `/` operator for old-style division | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __floordiv__, __rfloordiv__ | self, other | object | `//` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __truediv__, __rtruediv__ | self, other | object | `/` operator for new-style division | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __mod__, __rmod__ | self, other | object | `%` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __divmod__, __rdivmod__ | self, other | object | combined div and mod | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __pow__, __rpow__ | self, other, [mod] | object | `**` operator or pow(x, y, [mod]) | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __neg__ | self | object | unary `-` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __pos__ | self | object | unary `+` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __abs__ | self | object | absolute value | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __nonzero__ | self | int | convert to boolean | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __invert__ | self | object | `~` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __lshift__, __rlshift__ | self, other | object | `<<` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __rshift__, __rrshift__ | self, other | object | `>>` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __and__, __rand__ | self, other | object | `&` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __or__, __ror__ | self, other | object | `|` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __xor__, __rxor__ | self, other | object | `^` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ + +Note that Cython 0.x did not make use of the ``__r...__`` variants and instead +used the bidirectional C slot signature for the regular methods, thus making the +first argument ambiguous (not 'self' typed). +Since Cython 3.0, the operator calls are passed to the respective special methods. +See the section on :ref:`Arithmetic methods ` above. Numeric conversions ^^^^^^^^^^^^^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ @@ -261,12 +397,14 @@ +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __hex__ | self | object | Convert to hexadecimal | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __index__ (2.5+ only) | self | object | Convert to sequence index | +| __index__ | self | object | Convert to sequence index | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ In-place arithmetic operators ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ @@ -284,7 +422,7 @@ +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __imod__ | self, x | object | `%=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ipow__ | x, y, z | object | `**=` operator | +| __ipow__ | self, y, z | object | `**=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __ilshift__ | self, x | object | `<<=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -300,10 +438,12 @@ Sequences and mappings ^^^^^^^^^^^^^^^^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#emulating-container-types + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ -| __len__ | self int | | len(self) | +| __len__ | self | Py_ssize_t | len(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __getitem__ | self, x | object | self[x] | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -323,6 +463,8 @@ Iterators ^^^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#emulating-container-types + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ @@ -335,7 +477,7 @@ +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ -| __getbuffer__ | self, Py_buffer `*view`, int flags | | | +| __getbuffer__ | self, Py_buffer `*view`, int flags | | | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __releasebuffer__ | self, Py_buffer `*view` | | | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -346,7 +488,7 @@ +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ -| __getreadbuffer__ | self, Py_ssize_t i, void `**p` | | | +| __getreadbuffer__ | self, Py_ssize_t i, void `**p` | | | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __getwritebuffer__ | self, Py_ssize_t i, void `**p` | | | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -358,6 +500,8 @@ Descriptor objects (see note 2) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +https://docs.python.org/3/reference/datamodel.html#emulating-container-types + +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ diff -Nru cython-0.20.1+1~201611251650-6686/docs/src/userguide/wrapping_CPlusPlus.rst cython-0.20.1+1~202203241016-9537/docs/src/userguide/wrapping_CPlusPlus.rst --- cython-0.20.1+1~201611251650-6686/docs/src/userguide/wrapping_CPlusPlus.rst 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/src/userguide/wrapping_CPlusPlus.rst 2022-03-24 10:16:46.000000000 +0000 @@ -11,8 +11,8 @@ Cython has native support for most of the C++ language. Specifically: -* C++ objects can now be dynamically allocated with ``new`` and ``del`` keywords. -* C++ objects can be stack-allocated. +* C++ objects can be :term:`dynamically allocated` with ``new`` and ``del`` keywords. +* C++ objects can be :term:`stack-allocated`. * C++ classes can be declared with the new keyword ``cppclass``. * Templated classes and functions are supported. * Overloaded functions are supported. @@ -22,15 +22,14 @@ ------------------- The general procedure for wrapping a C++ file can now be described as follows: -* Specify C++ language in :file:`setup.py` script or locally in a source file. -* Create one or more .pxd files with ``cdef extern from`` blocks and - (if existing) the C++ namespace name. In these blocks, +* Specify C++ language in a :file:`setup.py` script or locally in a source file. +* Create one or more ``.pxd`` files with ``cdef extern from`` blocks and + (if existing) the C++ namespace name. In these blocks: * declare classes as ``cdef cppclass`` blocks * declare public names (variables, methods and constructors) -* Write an extension modules, ``cimport`` from the .pxd file and use - the declarations. +* ``cimport`` them in one or more extension modules (``.pyx`` files). A simple Tutorial ================== @@ -42,127 +41,18 @@ document. Let's assume it will be in a header file called :file:`Rectangle.h`: -.. sourcecode:: c++ - - namespace shapes { - class Rectangle { - public: - int x0, y0, x1, y1; - Rectangle(); - Rectangle(int x0, int y0, int x1, int y1); - ~Rectangle(); - int getArea(); - void getSize(int* width, int* height); - void move(int dx, int dy); - }; - } +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/Rectangle.h + :language: c++ + :tab-width: 4 and the implementation in the file called :file:`Rectangle.cpp`: -.. sourcecode:: c++ - - #include "Rectangle.h" - - namespace shapes { - - Rectangle::Rectangle() { } - - Rectangle::Rectangle(int X0, int Y0, int X1, int Y1) { - x0 = X0; - y0 = Y0; - x1 = X1; - y1 = Y1; - } - - Rectangle::~Rectangle() { } - - int Rectangle::getArea() { - return (x1 - x0) * (y1 - y0); - } - - void Rectangle::getSize(int *width, int *height) { - (*width) = x1 - x0; - (*height) = y1 - y0; - } - - void Rectangle::move(int dx, int dy) { - x0 += dx; - y0 += dy; - x1 += dx; - y1 += dy; - } - - } +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/Rectangle.cpp + :language: c++ + :tab-width: 4 This is pretty dumb, but should suffice to demonstrate the steps involved. -Specify C++ language in setup.py ---------------------------------- - -The best way to build Cython code from :file:`setup.py` scripts is the -``cythonize()`` function. To make Cython generate and compile C++ code -with distutils, you just need to pass the option ``language="c++"``:: - - from distutils.core import setup - from Cython.Build import cythonize - - setup(ext_modules = cythonize( - "rect.pyx", # our Cython source - sources=["Rectangle.cpp"], # additional source file(s) - language="c++", # generate C++ code - )) - -Cython will generate and compile the :file:`rect.cpp` file (from the -:file:`rect.pyx`), then it will compile :file:`Rectangle.cpp` -(implementation of the ``Rectangle`` class) and link both objects files -together into :file:`rect.so`, which you can then import in Python using -``import rect`` (if you forget to link the :file:`Rectangle.o`, you will -get missing symbols while importing the library in Python). - -Note that the ``language`` option has no effect on user provided Extension -objects that are passed into ``cythonize()``. It is only used for modules -found by file name (as in the example above). - -The ``cythonize()`` function in Cython versions up to 0.21 does not -recognize the ``language`` option and it needs to be specified as an -option to an :class:`Extension` that describes your extension and that -is then handled by ``cythonize()`` as follows:: - - from distutils.core import setup, Extension - from Cython.Build import cythonize - - setup(ext_modules = cythonize(Extension( - "rect", # the extension name - sources=["rect.pyx", "Rectangle.cpp"], # the Cython source and - # additional C++ source files - language="c++", # generate and compile C++ code - ))) - -The options can also be passed directly from the source file, which is -often preferable (and overrides any global option). Starting with -version 0.17, Cython also allows to pass external source files into the -``cythonize()`` command this way. Here is a simplified setup.py file:: - - from distutils.core import setup - from Cython.Build import cythonize - - setup( - name = "rectangleapp", - ext_modules = cythonize('*.pyx'), - ) - -And in the .pyx source file, write this into the first comment block, before -any source code, to compile it in C++ mode and link it statically against the -:file:`Rectangle.cpp` code file:: - - # distutils: language = c++ - # distutils: sources = Rectangle.cpp - -To compile manually (e.g. using ``make``), the ``cython`` command-line -utility can be used to generate a C++ ``.cpp`` file, and then compile it -into a python extension. C++ mode for the ``cython`` command is turned -on with the ``--cplus`` option. - Declaring a C++ class interface -------------------------------- @@ -173,7 +63,9 @@ cdef extern from "Rectangle.h" namespace "shapes": This will make the C++ class def for Rectangle available. Note the namespace declaration. -Namespaces are simply used to make the fully qualified name of the object, and can be nested (e.g. ``"outer::inner"``) or even refer to classes (e.g. ``"namespace::MyClass`` to declare static members on MyClass). +Namespaces are simply used to make the fully qualified name of the object, +and can be nested (e.g. ``"outer::inner"``) or even refer to +classes (e.g. ``"namespace::MyClass`` to declare static members on MyClass). Declare class with cdef cppclass ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -187,34 +79,52 @@ Add public attributes ^^^^^^^^^^^^^^^^^^^^^^ -We now need to declare the attributes and methods for use on Cython:: +We now need to declare the attributes and methods for use on Cython. We put those declarations +in a file called :file:`Rectangle.pxd`. You can see it as a header file +which is readable by Cython: - cdef extern from "Rectangle.h" namespace "shapes": - cdef cppclass Rectangle: - Rectangle() except + - Rectangle(int, int, int, int) except + - int x0, y0, x1, y1 - int getArea() - void getSize(int* width, int* height) - void move(int, int) +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/Rectangle.pxd -Note that the constructor is declared as "except +". If the C++ code or +Note that the constructor is declared as "except +". If the C++ code or the initial memory allocation raises an exception due to a failure, this will let Cython safely raise an appropriate Python exception instead (see below). Without this declaration, C++ exceptions originating from the constructor will not be handled by Cython. +We use the lines:: + + cdef extern from "Rectangle.cpp": + pass + +to include the C++ code from :file:`Rectangle.cpp`. It is also possible to specify to +setuptools that :file:`Rectangle.cpp` is a source. To do that, you can add this directive at the +top of the ``.pyx`` (not ``.pxd``) file:: + + # distutils: sources = Rectangle.cpp + +Note that when you use ``cdef extern from``, the path that you specify is relative to the current +file, but if you use the distutils directive, the path is relative to the +:file:`setup.py`. If you want to discover the path of the sources when +running the :file:`setup.py`, you can use the ``aliases`` argument +of the :func:`cythonize` function. + Declare a var with the wrapped C++ class ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now, we use cdef to declare a var of the class with the C++ ``new`` statement:: +We'll create a ``.pyx`` file named ``rect.pyx`` to build our wrapper. We're +using a name other than ``Rectangle``, but if you prefer giving the same name +to the wrapper as the C++ class, see the section on +:ref:`resolving naming conflicts `. - rec_ptr = new Rectangle(1, 2, 3, 4) - try: - recArea = rec_ptr.getArea() - ... - finally: - del rec_ptr # delete heap allocated object +Within, we use cdef to declare a var of the class with the C++ ``new`` statement: + +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/cython_usage.pyx + +The line:: + + # distutils: language = c++ + +is to indicate to Cython that this ``.pyx`` file has to be compiled to C++. It's also possible to declare a stack allocated object, as long as it has a "default" constructor:: @@ -226,6 +136,9 @@ def func(): cdef Foo foo ... + +See the section on the :ref:`cpp_locals directive` for a way +to avoid requiring a nullary/default constructor. Note that, like C++, if the class has only one constructor and it is a nullary one, it's not necessary to declare it. @@ -239,56 +152,47 @@ Common programming practice is to create a Cython extension type which holds a C++ instance as an attribute and create a bunch of -forwarding methods. So we can implement the Python extension type as:: +forwarding methods. So we can implement the Python extension type as: - cdef class PyRectangle: - cdef Rectangle c_rect # hold a C++ instance which we're wrapping - def __cinit__(self, int x0, int y0, int x1, int y1): - self.c_rect = Rectangle(x0, y0, x1, y1) - def get_area(self): - return self.c_rect.getArea() - def get_size(self): - cdef int width, height - self.c_rect.getSize(&width, &height) - return width, height - def move(self, dx, dy): - self.c_rect.move(dx, dy) +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/rect.pyx And there we have it. From a Python perspective, this extension type will look and feel just like a natively defined Rectangle class. -It should be noted that - -If you want to give -attribute access, you could just implement some properties:: +It should be noted that if you want to give +attribute access, you could just implement some properties: - @property - def x0(self): - return self.c_rect.x0 - - @x0.setter - def x0(self): - def __set__(self, x0): self.c_rect.x0 = x0 - ... +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/rect_with_attributes.pyx Cython initializes C++ class attributes of a cdef class using the nullary constructor. If the class you're wrapping does not have a nullary constructor, you must store a pointer -to the wrapped class and manually allocate and deallocate it. -A convienient and safe place to do so is in the `__cinit__` and `__dealloc__` methods +to the wrapped class and manually allocate and deallocate it. Alternatively, the +:ref:`cpp_locals directive` avoids the need for the pointer and only initializes the +C++ class attribute when it is assigned to. +A convenient and safe place to do so is in the `__cinit__` and `__dealloc__` methods which are guaranteed to be called exactly once upon creation and deletion of the Python instance. -:: +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx - cdef class PyRectangle: - cdef Rectangle* c_rect # hold a pointer to the C++ instance which we're wrapping - def __cinit__(self, int x0, int y0, int x1, int y1): - self.c_rect = new Rectangle(x0, y0, x1, y1) - def __dealloc__(self): - del self.c_rect - ... +Compilation and Importing +========================= + +To compile a Cython module, it is necessary to have a :file:`setup.py` file: -If you prefer giving the same name to the wrapper as the C++ class, see the -section on :ref:`resolving naming conflicts `. +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/setup.py + +Run ``$ python setup.py build_ext --inplace`` + +To test it, open the Python interpreter:: + + >>> import rect + >>> x0, y0, x1, y1 = 1, 2, 3, 4 + >>> rect_obj = rect.PyRectangle(x0, y0, x1, y1) + >>> print(dir(rect_obj)) + ['__class__', '__delattr__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', + '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', + '__lt__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', + '__setstate__', '__sizeof__', '__str__', '__subclasshook__', 'get_area', 'get_size', 'move'] Advanced C++ features @@ -321,6 +225,10 @@ Foo operator-(Foo) int operator*(Foo) int operator/(int) + int operator*(int, Foo) # allows 1*Foo() + # nonmember operators can also be specified outside the class + double operator/(double, Foo) + cdef Foo foo = new Foo() @@ -330,6 +238,13 @@ x = foo * foo2 x = foo / 1 + x = foo[0] * foo2 + x = foo[0] / 1 + x = 1*foo[0] + + cdef double y + y = 2.0/foo[0] + Note that if one has *pointers* to C++ objects, dereferencing must be done to avoid doing pointer arithmetic rather than arithmetic on the objects themselves:: @@ -344,30 +259,17 @@ Nested class declarations -------------------------- C++ allows nested class declaration. Class declarations can also be -nested in Cython:: - - cdef extern from "" namespace "std": - cdef cppclass vector[T]: - cppclass iterator: - T operator*() - iterator operator++() - bint operator==(iterator) - bint operator!=(iterator) - vector() - void push_back(T&) - T& operator[](int) - T& at(int) - iterator begin() - iterator end() +nested in Cython: - cdef vector[int].iterator iter #iter is declared as being of type vector::iterator +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/nested_class.pyx -Note that the nested class is declared with a ``cppclass`` but without a ``cdef``. +Note that the nested class is declared with a ``cppclass`` but without a ``cdef``, +as it is already part of a ``cdef`` declaration section. C++ operators not compatible with Python syntax ------------------------------------------------ -Cython try to keep a syntax as close as possible to standard Python. +Cython tries to keep its syntax as close as possible to standard Python. Because of this, certain C++ operators, like the preincrement ``++foo`` or the dereferencing operator ``*foo`` cannot be used with the same syntax as C++. Cython provides functions replacing these operators in @@ -391,36 +293,9 @@ Templates ---------- -Cython uses a bracket syntax for templating. A simple example for wrapping C++ vector:: - - # import dereference and increment operators - from cython.operator cimport dereference as deref, preincrement as inc +Cython uses a bracket syntax for templating. A simple example for wrapping C++ vector: - cdef extern from "" namespace "std": - cdef cppclass vector[T]: - cppclass iterator: - T operator*() - iterator operator++() - bint operator==(iterator) - bint operator!=(iterator) - vector() - void push_back(T&) - T& operator[](int) - T& at(int) - iterator begin() - iterator end() - - cdef vector[int] *v = new vector[int]() - cdef int i - for i in range(10): - v.push_back(i) - - cdef vector[int].iterator it = v.begin() - while it != v.end(): - print deref(it) - inc(it) - - del v +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/templates.pyx Multiple template parameters can be defined as a list, such as ``[T, U, V]`` or ``[int, bool, char]``. Optional template parameters can be indicated @@ -431,69 +306,57 @@ Template functions are defined similarly to class templates, with -the template parameter list following the function name:: - - cdef extern from "" namespace "std": - T max[T](T a, T b) +the template parameter list following the function name: - print max[long](3, 4) - print max(1.5, 2.5) # simple template argument deduction +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/function_templates.pyx Standard library ----------------- Most of the containers of the C++ Standard Library have been declared -in pxd files located in ``/Cython/Includes/libcpp``. These containers -are: deque, list, map, pair, queue, set, stack, vector. - -For example:: +in pxd files located +in `/Cython/Includes/libcpp `_. +These containers are: deque, list, map, pair, queue, set, stack, vector. - from libcpp.vector cimport vector +For example: - cdef vector[int] vect - cdef int i - for i in range(10): - vect.push_back(i) - for i in range(10): - print vect[i] +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/vector_demo.pyx -The pxd files in ``/Cython/Includes/libcpp`` also work as good examples on -how to declare C++ classes. +The pxd files +in `/Cython/Includes/libcpp `_ +also work as good examples on how to declare C++ classes. -Since Cython 0.17, the STL containers coerce from and to the +The STL containers coerce from and to the corresponding Python builtin types. The conversion is triggered either by an assignment to a typed variable (including typed function -arguments) or by an explicit cast, e.g.:: +arguments) or by an explicit cast, e.g.: - from libcpp.string cimport string - from libcpp.vector cimport vector - - cdef string s = py_bytes_object - print(s) - cpp_string = py_unicode_object.encode('utf-8') - - cdef vector[int] vect = xrange(1, 10, 2) - print(vect) # [1, 3, 5, 7, 9] - - cdef vector[string] cpp_strings = b'ab cd ef gh'.split() - print(cpp_strings[1]) # b'cd' +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx The following coercions are available: -+------------------+----------------+-----------------+ -| Python type => | *C++ type* | => Python type | -+==================+================+=================+ -| bytes | std::string | bytes | -+------------------+----------------+-----------------+ -| iterable | std::vector | list | -+------------------+----------------+-----------------+ -| iterable | std::list | list | -+------------------+----------------+-----------------+ -| iterable | std::set | set | -+------------------+----------------+-----------------+ -| iterable (len 2) | std::pair | tuple (len 2) | -+------------------+----------------+-----------------+ ++------------------+------------------------+-----------------+ +| Python type => | *C++ type* | => Python type | ++==================+========================+=================+ +| bytes | std::string | bytes | ++------------------+------------------------+-----------------+ +| iterable | std::vector | list | ++------------------+------------------------+-----------------+ +| iterable | std::list | list | ++------------------+------------------------+-----------------+ +| iterable | std::set | set | ++------------------+------------------------+-----------------+ +| iterable | std::unordered_set | set | ++------------------+------------------------+-----------------+ +| mapping | std::map | dict | ++------------------+------------------------+-----------------+ +| mapping | std::unordered_map | dict | ++------------------+------------------------+-----------------+ +| iterable (len 2) | std::pair | tuple (len 2) | ++------------------+------------------------+-----------------+ +| complex | std::complex | complex | ++------------------+------------------------+-----------------+ All conversions create a new container and copy the data into it. The items in the containers are converted to a corresponding type @@ -503,16 +366,21 @@ Iteration over stl containers (or indeed any class with ``begin()`` and ``end()`` methods returning an object supporting incrementing, dereferencing, and comparison) is supported via the ``for .. in`` syntax (including in list -comprehensions). For example, one can write:: +comprehensions). For example, one can write: - cdef vector[int] v = ... - for value in v: - f(value) - return [x*x for x in v if x % 2 == 0] +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/iterate.pyx If the loop target variable is unspecified, an assignment from type ``*container.begin()`` is used for :ref:`type inference `. +.. note:: + + Slicing stl containers is supported, + you can do ``for x in my_vector[:5]: ...`` but unlike pointers slices, + it will create a temporary Python object and iterate over it. Thus + making the iteration very slow. You might want to avoid slicing + C++ containers for performance reasons. + Simplified wrapping with default constructor -------------------------------------------- @@ -520,20 +388,9 @@ If your extension type instantiates a wrapped C++ class using the default constructor (not passing any arguments), you may be able to simplify the lifecycle handling by tying it directly to the lifetime of the Python wrapper -object. Instead of a pointer attribute, you can declare an instance:: - - cdef class VectorStack: - cdef vector[int] v - - def push(self, x): - self.v.push_back(x) +object. Instead of a pointer attribute, you can declare an instance: - def pop(self): - if self.v.empty(): - raise IndexError() - x = self.v.back() - self.v.pop_back() - return x +.. literalinclude:: ../../examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx Cython will automatically generate code that instantiates the C++ object instance when the Python object is created and deletes it when the Python @@ -598,12 +455,19 @@ raise_py_error does not actually raise an exception a RuntimeError will be raised. +There is also the special form:: + + cdef int raise_py_or_cpp() except +* + +for those functions that may raise either a Python or a C++ exception. + + Static member method -------------------- If the Rectangle class has a static member: -.. sourcecode:: c++ +.. code-block:: c++ namespace shapes { class Rectangle { @@ -631,6 +495,33 @@ functions as references (const or otherwise) as it has no impact on the caller's syntax. +Scoped Enumerations +------------------- + +Cython supports scoped enumerations (:keyword:`enum class`) in C++ mode:: + + cdef enum class Cheese: + cheddar = 1 + camembert = 2 + +As with "plain" enums, you may access the enumerators as attributes of the type. +Unlike plain enums however, the enumerators are not visible to the +enclosing scope:: + + cdef Cheese c1 = Cheese.cheddar # OK + cdef Cheese c2 = cheddar # ERROR! + +Optionally, you may specify the underlying type of a scoped enumeration. +This is especially important when declaring an external scoped enumeration +with an underlying type:: + + cdef extern from "Foo.h": + cdef enum class Spam(unsigned int): + x = 10 + y = 20 + ... + +Declaring an enum class as ``cpdef`` will create a :pep:`435`-style Python wrapper. ``auto`` Keyword ---------------- @@ -645,7 +536,7 @@ cdef vector[int] v = ... it = v.begin() -(Though of course the ``for .. in`` syntax is prefered for objects supporting +(Though of course the ``for .. in`` syntax is preferred for objects supporting the iteration protocol.) RTTI and typeid() @@ -658,7 +549,7 @@ The ``typeid(...)`` operator returns an object of the type ``const type_info &``. If you want to store a type_info value in a C variable, you will need to store it -as a pointer rather than a reference: +as a pointer rather than a reference:: from libcpp.typeinfo cimport type_info cdef const type_info* info = &typeid(MyClass) @@ -670,6 +561,118 @@ in ``libcpp.typeindex``. +Specify C++ language in setup.py +================================ + +Instead of specifying the language and the sources in the source files, it is +possible to declare them in the :file:`setup.py` file:: + + from setuptools import setup + from Cython.Build import cythonize + + setup(ext_modules = cythonize( + "rect.pyx", # our Cython source + sources=["Rectangle.cpp"], # additional source file(s) + language="c++", # generate C++ code + )) + +Cython will generate and compile the :file:`rect.cpp` file (from +:file:`rect.pyx`), then it will compile :file:`Rectangle.cpp` +(implementation of the ``Rectangle`` class) and link both object files +together into :file:`rect.so` on Linux, or :file:`rect.pyd` on windows, +which you can then import in Python using +``import rect`` (if you forget to link the :file:`Rectangle.o`, you will +get missing symbols while importing the library in Python). + +Note that the ``language`` option has no effect on user provided Extension +objects that are passed into ``cythonize()``. It is only used for modules +found by file name (as in the example above). + +The ``cythonize()`` function in Cython versions up to 0.21 does not +recognize the ``language`` option and it needs to be specified as an +option to an :class:`Extension` that describes your extension and that +is then handled by ``cythonize()`` as follows:: + + from setuptools import Extension, setup + from Cython.Build import cythonize + + setup(ext_modules = cythonize(Extension( + "rect", # the extension name + sources=["rect.pyx", "Rectangle.cpp"], # the Cython source and + # additional C++ source files + language="c++", # generate and compile C++ code + ))) + +The options can also be passed directly from the source file, which is +often preferable (and overrides any global option). Starting with +version 0.17, Cython also allows passing external source files into the +``cythonize()`` command this way. Here is a simplified setup.py file:: + + from setuptools import setup + from Cython.Build import cythonize + + setup( + name = "rectangleapp", + ext_modules = cythonize('*.pyx'), + ) + +And in the .pyx source file, write this into the first comment block, before +any source code, to compile it in C++ mode and link it statically against the +:file:`Rectangle.cpp` code file:: + + # distutils: language = c++ + # distutils: sources = Rectangle.cpp + +.. note:: + + When using distutils directives, the paths are relative to the working + directory of the setuptools run (which is usually the project root where + the :file:`setup.py` resides). + +To compile manually (e.g. using ``make``), the ``cython`` command-line +utility can be used to generate a C++ ``.cpp`` file, and then compile it +into a python extension. C++ mode for the ``cython`` command is turned +on with the ``--cplus`` option. + +.. _cpp_locals directive: + +``cpp_locals`` directive +======================== + +The ``cpp_locals`` compiler directive is an experimental feature that makes +C++ variables behave like normal Python object variables. With this +directive they are only initialized at their first assignment, and thus +they no longer require a nullary constructor to be stack-allocated. Trying to +access an uninitialized C++ variable will generate an ``UnboundLocalError`` +(or similar) in the same way as a Python variable would. For example:: + + def function(dont_write): + cdef SomeCppClass c # not initialized + if dont_write: + return c.some_cpp_function() # UnboundLocalError + else: + c = SomeCppClass(...) # initialized + return c.some_cpp_function() # OK + +Additionally, the directive avoids initializing temporary C++ objects before +they are assigned, for cases where Cython needs to use such objects in its +own code-generation (often for return values of functions that can throw +exceptions). + +For extra speed, the ``initializedcheck`` directive disables the check for an +unbound-local. With this directive on, accessing a variable that has not +been initialized will trigger undefined behaviour, and it is entirely the user's +responsibility to avoid such access. + +The ``cpp_locals`` directive is currently implemented using ``std::optional`` +and thus requires a C++17 compatible compiler. Defining +``CYTHON_USE_BOOST_OPTIONAL`` (as define for the C++ compiler) uses ``boost::optional`` +instead (but is even more experimental and untested). The directive may +come with a memory and performance cost due to the need to store and check +a boolean that tracks if a variable is initialized, but the C++ compiler should +be able to eliminate the check in most cases. + + Caveats and Limitations ======================== diff -Nru cython-0.20.1+1~201611251650-6686/docs/_static/css/tabs.css cython-0.20.1+1~202203241016-9537/docs/_static/css/tabs.css --- cython-0.20.1+1~201611251650-6686/docs/_static/css/tabs.css 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/_static/css/tabs.css 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,60 @@ +.sphinx-tabs { + margin-bottom: 1rem; +} + +[role="tablist"] { + border-bottom: 0px solid #a0b3bf; +} + +.sphinx-tabs-tab { + position: relative; + font-family: 'Helvetica Neue',Arial,Helvetica,sans-serif; + color: black; + line-height: 24px; + margin: 0; + font-size: 16px; + font-weight: 400; + background-color: rgba(255, 255, 255, 0); + border-radius: 5px 5px 5px 5px; + border: 0; + padding: 0.5rem 1.6rem; + margin-bottom: 0; +} + +.sphinx-tabs-tab[aria-selected="true"] { + border: 2px solid gray; + border-bottom: 2px solid gray; + margin: -2px; + background-color: #efefef; + z-index: 999; /* render on top*/ +} + +.sphinx-tabs-tab[aria-selected="false"] { + border: 2px solid #dddddd; + border-bottom: 2px solid #dddddd; + margin: -2px; + background-color: white; + } + +.sphinx-tabs-tab:focus { + z-index: 1; + outline-offset: 1px; +} + +.sphinx-tabs-panel { + position: relative; + padding: 0rem; + border: 0px solid #a0b3bf; + margin: 0px 0px 0px 0px; + border-radius: 0 0 5px 5px; + border-top: 0; + background: white; +} + +.sphinx-tabs-panel.code-tab { + padding: 0.4rem; +} + +.sphinx-tab img { + margin-bottom: 24 px; +} diff -Nru cython-0.20.1+1~201611251650-6686/docs/_templates/layout.html cython-0.20.1+1~202203241016-9537/docs/_templates/layout.html --- cython-0.20.1+1~201611251650-6686/docs/_templates/layout.html 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/_templates/layout.html 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,28 @@ {% extends "!layout.html" %} +{% block header %} +{%- if development %} +
    + {% trans %}This version of the documentation is for the latest and greatest in-development branch of Cython. + For the last release version, see {% endtrans %} + {% trans %}here{% endtrans %}. +
    +{%- endif %} +{% endblock %} + +{% block relbar1 %} +{{ super() }} +{% if pagename != "src/donating" %} +
    +

    🤝 Like the tool? Help making it better! Your donation helps! 🤝

    +
    +{% endif %} +{% endblock %} + + {% block footer %} {{ super() }} +{# ## Removed to avoid tracking our users. +#} {% endblock %} diff -Nru cython-0.20.1+1~201611251650-6686/docs/TODO cython-0.20.1+1~202203241016-9537/docs/TODO --- cython-0.20.1+1~201611251650-6686/docs/TODO 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/docs/TODO 2022-03-24 10:16:46.000000000 +0000 @@ -6,21 +6,21 @@ The "Old Cython Users Guide" is a derivative of the old Pyrex documentation. It underwent substantial editing by Peter Alexandar to become the Reference Guide, which is oriented around bullet points -and lists rather than prose. This transition was incomplete. +and lists rather than prose. This transition was incomplete. -At nearly the same time, Robert, Dag, and Stefan wrote a tutorial as -part of the SciPy proceedings. It was felt that the content there was -cleaner and more up to date than anything else, and this became the -basis for the "Getting Started" and "Tutorials" sections. However, -it simply doesn't have as much content as the old documentation used to. +At nearly the same time, Robert, Dag, and Stefan wrote a tutorial as +part of the SciPy proceedings. It was felt that the content there was +cleaner and more up to date than anything else, and this became the +basis for the "Getting Started" and "Tutorials" sections. However, +it simply doesn't have as much content as the old documentation used to. Eventually, it seems all of the old users manual could be whittled down into independent tutorial topics. Much discussion of what we'd -like to see is at +like to see is at -http://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html +https://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html -There is currently a huge amount of redundancy, but no one section has -it all. +There is currently a huge amount of redundancy, but no one section has +it all. Also, we should go through the wiki enhancement proposal list -and make sure to transfer the (done) ones into the user manual. +and make sure to transfer the (done) ones into the user manual. diff -Nru cython-0.20.1+1~201611251650-6686/.editorconfig cython-0.20.1+1~202203241016-9537/.editorconfig --- cython-0.20.1+1~201611251650-6686/.editorconfig 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.editorconfig 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,23 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +max_line_length = 120 + +# 4 space indentation +[*.{py,pyx,pxi,pxd,c,cpp,h,hpp}] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +# Matches the exact files either package.json or .travis.yml +[{package.json,.travis.yml}] +indent_style = space +indent_size = 2 diff -Nru cython-0.20.1+1~201611251650-6686/.github/code-of-conduct.md cython-0.20.1+1~202203241016-9537/.github/code-of-conduct.md --- cython-0.20.1+1~201611251650-6686/.github/code-of-conduct.md 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/code-of-conduct.md 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,78 @@ +# Cython Code of Conduct + +## Introduction + +This Code of Conduct applies to all spaces managed by the Cython project, including all public and private mailing lists, issue trackers, wikis, and any other communication channel used by our community. The Cython project may also organise or participate in in-person or virtual events. This Code of Conduct applies to events organized by the Cython project, and we expect other events related to our community to have a code of conduct similar in spirit to this one. + +This Code of Conduct should be honored by everyone who participates in the Cython community formally or informally, or claims any affiliation with the project, in any project-related activities and especially when representing the project, in any role. + +This code is not exhaustive or complete. It serves to distill our common understanding of a collaborative, shared environment and goals. Please try to follow this code in spirit as much as in letter, to create a friendly and productive environment that enriches the surrounding community. + +## Specific Guidelines + +We strive to: + +1. Be open. We invite anyone to participate in our community. We prefer to use public methods of communication for project-related messages, unless discussing something sensitive. This applies to messages for help or project-related support, too; not only is a public support request much more likely to result in an answer to a question, it also ensures that any inadvertent mistakes in answering are more easily detected and corrected. +2. Be empathetic, welcoming, friendly, and patient. We work together to resolve conflict, and assume good intentions. We may all experience some frustration from time to time, but we do not allow frustration to turn into a personal attack. A community where people feel uncomfortable or threatened is not a productive one. +3. Be collaborative. Our work will be used by other people, and in turn we will depend on the work of others. When we make something for the benefit of the project, we are willing to explain to others how it works, so that they can build on the work to make it even better. Any decision we make will affect users and colleagues, and we take those consequences seriously when making decisions. +4. Be inquisitive. Nobody knows everything! Asking questions early avoids many problems later, so we encourage questions, although we may direct them to the appropriate forum. We will try hard to be responsive and helpful. +5. Be careful in the words that we choose. We are careful and respectful in our communication, and we take responsibility for our own speech. Be kind to others. Do not insult or put down other participants. We will not accept harassment or other exclusionary behaviour, such as: + * Violent threats or language directed against another person. + * Sexist, racist, or otherwise discriminatory jokes and language. + * Posting sexually explicit or violent material. + * Posting (or threatening to post) other people’s personally identifying information (“doxing”). + * Sharing private content, such as emails sent privately or non-publicly, or unlogged forums such as IRC channel history, without the sender’s consent. + * Personal insults, especially those using racist or sexist terms. + * Unwelcome sexual attention. + * Excessive profanity. Please avoid swearwords; people differ greatly in their sensitivity to swearing. + * Repeated harassment of others. In general, if someone asks you to stop, then stop. + * Advocating for, or encouraging, any of the above behaviour. + +## Diversity Statement + +The Cython project welcomes and encourages participation by everyone. We are committed to being a community that everyone enjoys being part of. Although we may not always be able to accommodate each individual’s preferences, we try our best to treat everyone kindly. + +No matter how you identify yourself or how others perceive you: we welcome you. Though no list can hope to be comprehensive, we explicitly honour diversity in: age, culture, ethnicity, genotype, gender identity or expression, language, national origin, neurotype, phenotype, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, subculture and technical ability, to the extent that these do not conflict with this code of conduct. + +Though we welcome people fluent in all languages, Cython development is conducted in English. + +Standards for behaviour in the Cython community are detailed in the Code of Conduct above. Participants in our community should uphold these standards in all their interactions and help others to do so as well (see next section). + +## Reporting Guidelines + +We know that it is painfully common for Internet communication to start at or devolve into obvious and flagrant abuse. We also recognize that sometimes people may have a bad day, or be unaware of some of the guidelines in this Code of Conduct. Please keep this in mind when deciding on how to respond to a breach of this Code. + +For clearly intentional breaches, report those to the Code of Conduct Committee (see below). For possibly unintentional breaches, you may reply to the person and point out this code of conduct (either in public or in private, whatever is most appropriate). If you would prefer not to do that, please feel free to report to the Code of Conduct Committee directly, or ask the Committee for advice, in confidence. + +You can report issues to the Cython Code of Conduct Committee at cython-conduct@googlegroups.com. + +Currently, the Committee consists of: + +* Stefan Behnel +* Robert Bradshaw +* Ralf Gommers + +If your report involves any members of the Committee, or if they feel they have a conflict of interest in handling it, then they will step aside and not involve themselves from considering your report. Alternatively, if for any reason you feel uncomfortable making a report to the Committee, then you can also contact senior NumFOCUS staff at [conduct@numfocus.org](https://numfocus.org/code-of-conduct#persons-responsible). + +## Incident reporting resolution & Code of Conduct enforcement + +_This section summarizes the most important points, more details can be found in_ [Cython Code of Conduct - How to follow up on a report](report-handling-manual.md). + +We will investigate and respond to all complaints. The Cython Code of Conduct Committee will protect the identity of the reporter, and treat the content of complaints as confidential (unless the reporter agrees otherwise). + +In case of severe and obvious breaches, e.g. personal threat or violent, sexist or racist language, we will immediately disconnect the originator from Cython communication channels; please see the manual for details. + +In cases not involving clear severe and obvious breaches of this Code of Conduct the process for acting on any received Code of Conduct violation report will be: + +1. acknowledge report is received, +2. reasonable discussion/feedback, +3. mediation (if feedback didn’t help, and only if both reporter and reportee agree to this), +4. enforcement via transparent decision (see [Resolutions](report-handling-manual.md#resolutions)) by the Code of Conduct Committee. + +The Committee will respond to any report as soon as possible, and at most within 72 hours. + +## Endnotes + +We are thankful to the groups behind the following documents, from which we drew content and inspiration: + +- [The SciPy Code of Conduct](https://docs.scipy.org/doc/scipy/reference/dev/conduct/code_of_conduct.html) diff -Nru cython-0.20.1+1~201611251650-6686/.github/FUNDING.yml cython-0.20.1+1~202203241016-9537/.github/FUNDING.yml --- cython-0.20.1+1~201611251650-6686/.github/FUNDING.yml 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/FUNDING.yml 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: scoder # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: pypi/Cython # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff -Nru cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/bug_report.md cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/bug_report.md --- cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/bug_report.md 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/bug_report.md 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,39 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG] " +labels: '' +assignees: '' + +--- + + + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Code to reproduce the behaviour: +```cython +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment (please complete the following information):** + - OS: [e.g. Linux, Windows, macOS] + - Python version [e.g. 3.8.4] + - Cython version [e.g. 0.29.18] + +**Additional context** +Add any other context about the problem here. diff -Nru cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/config.yml cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/config.yml --- cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/config.yml 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/config.yml 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Cython-users mailing list + url: https://groups.google.com/g/cython-users + about: Please ask and answer support questions here instead of on github. diff -Nru cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/feature_request.md cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/feature_request.md --- cython-0.20.1+1~201611251650-6686/.github/ISSUE_TEMPLATE/feature_request.md 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/ISSUE_TEMPLATE/feature_request.md 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,41 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[ENH] " +labels: '' +assignees: '' + +--- + + + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. In my code, I would like to do [...] +```cython +# add use case related code here +``` + +**Describe the solution you'd like** +A clear and concise description of what you want to happen, including code examples if applicable. +```cython +# add a proposed code/syntax example here +``` + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. +```cython +# add alternative code/syntax proposals here +``` + +**Additional context** +Add any other context about the feature request here. diff -Nru cython-0.20.1+1~201611251650-6686/.github/report-handling-manual.md cython-0.20.1+1~202203241016-9537/.github/report-handling-manual.md --- cython-0.20.1+1~201611251650-6686/.github/report-handling-manual.md 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/report-handling-manual.md 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,92 @@ +# Cython Code of Conduct - How to follow up on a report + +This is the manual followed by Cython’s Code of Conduct Committee. It’s used when we respond to an issue to make sure we’re consistent and fair. + +Enforcing the [Code of Conduct](code-of-conduct.md) impacts our community today and for the future. It’s an action that we do not take lightly. When reviewing enforcement measures, the Code of Conduct Committee will keep the following values and guidelines in mind: + +* Act in a personal manner rather than impersonal. The Committee can engage the parties to understand the situation while respecting the privacy and any necessary confidentiality of reporters. However, sometimes it is necessary to communicate with one or more individuals directly: the Committee’s goal is to improve the health of our community rather than only produce a formal decision. +* Emphasize empathy for individuals rather than judging behavior, avoiding binary labels of “good” and “bad/evil”. Overt, clear-cut aggression and harassment exist, and we will address them firmly. But many scenarios that can prove challenging to resolve are those where normal disagreements devolve into unhelpful or harmful behavior from multiple parties. Understanding the full context and finding a path that re-engages all is hard, but ultimately the most productive for our community. +* We understand that email is a difficult medium and can be isolating. Receiving criticism over email, without personal contact, can be particularly painful. This makes it especially important to keep an atmosphere of open-minded respect for the views of others. It also means that we must be transparent in our actions, and that we will do everything in our power to make sure that all our members are treated fairly and with sympathy. +* Discrimination can be subtle and it can be unconscious. It can show itself as unfairness and hostility in otherwise ordinary interactions. We know that this does occur, and we will take care to look out for it. We would very much like to hear from you if you feel you have been treated unfairly, and we will use these procedures to make sure that your complaint is heard and addressed. +* Help increase engagement in good discussion practice: try to identify where discussion may have broken down, and provide actionable information, pointers, and resources that can lead to positive change on these points. +* Be mindful of the needs of new members: provide them with explicit support and consideration, with the aim of increasing participation from underrepresented groups in particular. +* Individuals come from different cultural backgrounds and native languages. Try to identify any honest misunderstandings caused by a non-native speaker and help them understand the issue and what they can change to avoid causing offence. Complex discussion in a foreign language can be very intimidating, and we want to grow our diversity also across nationalities and cultures. + + +## Mediation + +Voluntary informal mediation is a tool at our disposal. In contexts such as when two or more parties have all escalated to the point of inappropriate behavior (something sadly common in human conflict), it may be useful to facilitate a mediation process. This is only an example: the Committee can consider mediation in any case, mindful that the process is meant to be strictly voluntary and no party can be pressured to participate. If the Committee suggests mediation, it should: + +* Find a candidate who can serve as a mediator. +* Obtain the agreement of the reporter(s). The reporter(s) have complete freedom to decline the mediation idea or to propose an alternate mediator. +* Obtain the agreement of the reported person(s). +* Settle on the mediator: while parties can propose a different mediator than the suggested candidate, only if a common agreement is reached on all terms can the process move forward. +* Establish a timeline for mediation to complete, ideally within two weeks. + +The mediator will engage with all the parties and seek a resolution that is satisfactory to all. Upon completion, the mediator will provide a report (vetted by all parties to the process) to the Committee, with recommendations on further steps. The Committee will then evaluate these results (whether a satisfactory resolution was achieved or not) and decide on any additional action deemed necessary. + + +## How the Committee will respond to reports + +When the Committee (or a Committee member) receives a report, they will first determine whether the report is about a clear and severe breach (as defined below). If so, immediate action needs to be taken in addition to the regular report handling process. + + +## Clear and severe breach actions + +We know that it is painfully common for internet communication to start at or devolve into obvious and flagrant abuse. We will deal quickly with clear and severe breaches like personal threats, violent, sexist or racist language. + +When a member of the Code of Conduct Committee becomes aware of a clear and severe breach, they will do the following: + +* Immediately disconnect the originator from all Cython communication channels. +* Reply to the reporter that their report has been received and that the originator has been disconnected. +* In every case, the moderator should make a reasonable effort to contact the originator, and tell them specifically how their language or actions qualify as a “clear and severe breach”. The moderator should also say that, if the originator believes this is unfair or they want to be reconnected to Cython, they have the right to ask for a review, as below, by the Code of Conduct Committee. The moderator should copy this explanation to the Code of Conduct Committee. +* The Code of Conduct Committee will formally review and sign off on all cases where this mechanism has been applied to make sure it is not being used to control ordinary heated disagreement. + + +## Report handling + +When a report is sent to the Committee they will immediately reply to the reporter to confirm receipt. This reply must be sent within 72 hours, and the group should strive to respond much quicker than that. + +If a report doesn’t contain enough information, the Committee will obtain all relevant data before acting. It may contact any individuals involved to get a more complete account of events. + +The Committee will then review the incident and determine, to the best of their ability: + +* What happened. +* Whether this event constitutes a Code of Conduct violation. +* Who are the responsible party(ies). +* Whether this is an ongoing situation, and there is a threat to anyone’s physical safety. + +This information will be collected in writing, and whenever possible the group’s deliberations will be recorded and retained (i.e. chat transcripts, email discussions, recorded conference calls, summaries of voice conversations, etc). + +It is important to retain an archive of all activities of this Committee to ensure consistency in behavior and provide institutional memory for the project. To assist in this, the default channel of discussion for this Committee will be a private mailing list accessible to current and future members of the Committee. If the Committee finds the need to use off-list communications (e.g. phone calls for early/rapid response), it should in all cases summarize these back to the list so there’s a good record of the process. + +The Code of Conduct Committee should aim to have a resolution agreed upon within two weeks. In the event that a resolution can’t be determined in that time, the Committee will respond to the reporter(s) with an update and projected timeline for resolution. + + +## Resolutions + +The Committee must agree on a resolution by consensus. If the group cannot reach consensus and deadlocks for over a week, the group will turn the matter over to the NumFOCUS Code of Conduct Enforcement Team for resolution. + +Possible responses may include: + +* Taking no further action: + - if we determine no violations have occurred; + - if the matter has been resolved publicly while the Committee was considering responses. +* Coordinating voluntary mediation: if all involved parties agree, the Committee may facilitate a mediation process as detailed above. +* Remind publicly, and point out that some behavior/actions/language have been judged inappropriate and why in the current context, or can but hurtful to some people, requesting the community to self-adjust. +* A private reprimand from the Committee to the individual(s) involved. In this case, the group chair will deliver that reprimand to the individual(s) over email, cc’ing the group. +* A public reprimand. In this case, the Committee chair will deliver that reprimand in the same venue that the violation occurred, within the limits of practicality. E.g., the original mailing list for an email violation, but for a chat room discussion where the person/context may be gone, they can be reached by other means. The group may choose to publish this message elsewhere for documentation purposes. +* A request for a public or private apology, assuming the reporter agrees to this idea: they may at their discretion refuse further contact with the violator. The chair will deliver this request. The Committee may, if it chooses, attach “strings” to this request: for example, the group may ask a violator to apologize in order to retain one’s membership on a mailing list. +* A “mutually agreed upon hiatus” where the Committee asks the individual to temporarily refrain from community participation. If the individual chooses not to take a temporary break voluntarily, the Committee may issue a “mandatory cooling off period”. +* A permanent or temporary ban from some or all Cython spaces (mailing list, GitHub, etc.). The group will maintain records of all such bans so that they may be reviewed in the future or otherwise maintained. + +Once a resolution is agreed upon, but before it is enacted, the Committee will contact the original reporter and any other affected parties and explain the proposed resolution. The Committee will ask if this resolution is acceptable, and must note feedback for the record. + +Finally, the Committee will make a report to the Cython project leadership (as well as the Cython core team in the event of an ongoing resolution, such as a ban). + +The Committee will never publicly discuss the issue; all public statements will be made by the chair of the Code of Conduct Committee. + + +## Conflicts of Interest + +In the event of any conflict of interest, a Committee member must immediately notify the other members, and recuse themselves if necessary. diff -Nru cython-0.20.1+1~201611251650-6686/.github/workflows/ci.yml cython-0.20.1+1~202203241016-9537/.github/workflows/ci.yml --- cython-0.20.1+1~201611251650-6686/.github/workflows/ci.yml 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/workflows/ci.yml 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,299 @@ +name: CI + +on: [push, pull_request] + +jobs: + ci: + strategy: + # Allows for matrix sub-jobs to fail without canceling the rest + fail-fast: false + + # MATRIX: + # ======= + # Required parameters: + # os the os to run on + # python-version the python version to use + # backend the backend to use + # env any additional env variables. Set to '{}' for none + # Optional parameters: + # allowed_failure whether the job is allowed to fail + # extra_hash extra hash str to differentiate from other caches with similar name (must always start with '-') + matrix: + # Tests [amd64] + # + # FIXME: 'cpp' tests seems to fail due to compilation errors (numpy_pythran_unit) + # in all python versions and test failures (builtin_float) in 3.5< + os: [ubuntu-18.04] + backend: [c, cpp] + python-version: ["2.7", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev"] + env: [{}] + + include: + # Temporary - Allow failure on Python 3.11-dev jobs until they are considered stable + - python-version: 3.11-dev + allowed_failure: true + + # Ubuntu sub-jobs: + # ================ + # GCC 11 + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + # compile all modules + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 2.7 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + # Linting + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { TEST_CODE_STYLE: 1, NO_CYTHON_COMPILE: 1 } + extra_hash: "-codestyle" + # Limited API + - os: ubuntu-18.04 + python-version: 3.6 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.8 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + # Type specs + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.8 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.7 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + # Stackless + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { STACKLESS: true, PY: 2 } + extra_hash: "-stackless" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { STACKLESS: true, PY: 3 } + extra_hash: "-stackless" + - os: ubuntu-18.04 + python-version: 3.8 + backend: c + env: { STACKLESS: true, PY: 3 } + extra_hash: "-stackless" + # Pypy + - os: ubuntu-18.04 + python-version: pypy-2.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + - os: ubuntu-18.04 + python-version: pypy-3.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + # Coverage - Disabled due to taking too long to run + # - os: ubuntu-18.04 + # python-version: 3.7 + # backend: "c,cpp" + # env: { COVERAGE: 1 } + # extra_hash: '-coverage' + + # MacOS sub-jobs + # ============== + # (C-only builds are used to create wheels) + - os: macos-10.15 + python-version: 2.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 2.7 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.5 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.6 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.8 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: "3.10" + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: "3.10" + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + + # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. + # From testing, the runs tend to take ~20/~30 minutes, so a limit of 40 minutes should be enough. This can always be + # changed in the future if needed. + timeout-minutes: 40 + runs-on: ${{ matrix.os }} + + env: + BACKEND: ${{ matrix.backend }} + OS_NAME: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + GCC_VERSION: 8 + USE_CCACHE: 1 + CCACHE_SLOPPINESS: "pch_defines,time_macros" + CCACHE_COMPRESS: 1 + CCACHE_MAXSIZE: "200M" + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache [ccache] + uses: pat-s/always-upload-cache@v2.1.3 + if: startsWith(runner.os, 'Linux') + with: + path: ~/.ccache + key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ matrix.backend == 'c' || matrix.backend == 'c,cpp' }}-${{ contains(matrix.backend, 'cpp') }}-${{ hashFiles('**/test-requirements*.txt', '**/ci.yml', '**/ci-run.sh') }} + + - name: Run CI + continue-on-error: ${{ matrix.allowed_failure || false }} + env: ${{ matrix.env }} + run: bash ./Tools/ci-run.sh + + - name: Upload HTML docs + uses: actions/upload-artifact@v2 + with: + name: htmldocs + path: docs/build/html + if-no-files-found: ignore + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-${{ runner.os }} + path: dist/*.whl + if-no-files-found: ignore + + + pycoverage: + runs-on: ubuntu-18.04 + + env: + BACKEND: c,cpp + OS_NAME: ubuntu-18.04 + PYTHON_VERSION: 3.9 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Run Coverage + env: { COVERAGE: 1, NO_CYTHON_COMPILE: 1 } + run: bash ./Tools/ci-run.sh + + - name: Upload Coverage Report + uses: actions/upload-artifact@v2 + with: + name: pycoverage_html + path: coverage-report-html + +# cycoverage: +# runs-on: ubuntu-18.04 +# +# env: +# BACKEND: c,cpp +# OS_NAME: ubuntu-18.04 +# PYTHON_VERSION: 3.9 +# +# steps: +# - name: Checkout repo +# uses: actions/checkout@v2 +# with: +# fetch-depth: 1 +# +# - name: Setup python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.9 +# +# - name: Run Coverage +# env: { COVERAGE: 1 } +# run: bash ./Tools/ci-run.sh +# +# - name: Upload Coverage Report +# uses: actions/upload-artifact@v2 +# with: +# name: cycoverage_html +# path: coverage-report-html diff -Nru cython-0.20.1+1~201611251650-6686/.github/workflows/wheel-manylinux.yml cython-0.20.1+1~202203241016-9537/.github/workflows/wheel-manylinux.yml --- cython-0.20.1+1~201611251650-6686/.github/workflows/wheel-manylinux.yml 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/.github/workflows/wheel-manylinux.yml 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,90 @@ +name: Linux wheel build + +on: + release: + types: [created] + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install build dependencies + run: pip install -U "setuptools<60" pip wheel + + - name: Make sdist and Python wheel + run: make sdist pywheel + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + dist/*.tar.gz + dist/*-none-any.whl + + - name: Upload sdist + uses: actions/upload-artifact@v2 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: ignore + + - name: Upload Python wheel + uses: actions/upload-artifact@v2 + with: + name: wheel-Python + path: dist/*-none-any.whl + if-no-files-found: ignore + + binary: + strategy: + # Allows for matrix sub-jobs to fail without canceling the rest + fail-fast: false + + matrix: + image: + - manylinux1_x86_64 + - manylinux1_i686 + - musllinux_1_1_x86_64 + - manylinux_2_24_x86_64 + - manylinux_2_24_i686 + - manylinux_2_24_aarch64 + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Building wheel + run: | + make sdist wheel_${{ matrix.image }} + + - name: Copy wheels in dist + run: cp wheelhouse*/*.whl dist/ + + - name: Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + dist/*manylinux*.whl + dist/*musllinux*.whl + + - name: Archive Wheels + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.image }} + path: dist/*m[au][ns][yl]linux*.whl + if-no-files-found: ignore diff -Nru cython-0.20.1+1~201611251650-6686/LICENSE.txt cython-0.20.1+1~202203241016-9537/LICENSE.txt --- cython-0.20.1+1~201611251650-6686/LICENSE.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/LICENSE.txt 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION diff -Nru cython-0.20.1+1~201611251650-6686/Makefile cython-0.20.1+1~202203241016-9537/Makefile --- cython-0.20.1+1~201611251650-6686/Makefile 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/Makefile 2022-03-24 10:16:46.000000000 +0000 @@ -1,11 +1,40 @@ +PACKAGENAME=Cython PYTHON?=python TESTOPTS?= REPO = git://github.com/cython/cython.git +VERSION?=$(shell sed -ne 's|^__version__\s*=\s*"\([^"]*\)".*|\1|p' Cython/Shadow.py) +PARALLEL?=$(shell ${PYTHON} -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")' || true) -all: local +MANYLINUX_CFLAGS=-O3 -g0 -mtune=generic -pipe -fPIC +MANYLINUX_LDFLAGS= +MANYLINUX_IMAGES= \ + manylinux1_x86_64 \ + manylinux1_i686 \ + musllinux_1_1_x86_64 \ + manylinux_2_24_x86_64 \ + manylinux_2_24_i686 \ + manylinux_2_24_aarch64 \ +# manylinux_2_24_ppc64le \ +# manylinux_2_24_s390x + +all: local local: - ${PYTHON} setup.py build_ext --inplace + ${PYTHON} setup.py build_ext --inplace $(PARALLEL) + +plocal: + ${PYTHON} setup.py build_ext --inplace --cython-profile $(PARALLEL) + +sdist: dist/$(PACKAGENAME)-$(VERSION).tar.gz + +dist/$(PACKAGENAME)-$(VERSION).tar.gz: + $(PYTHON) setup.py sdist + +pywheel: dist/$(PACKAGENAME)-$(VERSION)-py2.py3-none-any.whl + +dist/$(PACKAGENAME)-$(VERSION)-py2.py3-none-any.whl: + ${PYTHON} setup.py bdist_wheel --no-cython-compile --universal + [ -f "$@" ] # check that we generated the expected universal wheel TMPDIR = .repo_tmp .git: .gitrev @@ -16,6 +45,7 @@ rm -rf $(TMPDIR) git ls-files -d | xargs git checkout -- +# Create a git repo from an unpacked source directory. repo: .git @@ -23,10 +53,11 @@ @echo Cleaning Source @rm -fr build @rm -f *.py[co] */*.py[co] */*/*.py[co] */*/*/*.py[co] - @rm -f *.so */*.so */*/*.so - @rm -f *.pyd */*.pyd */*/*.pyd + @rm -f *.so */*.so */*/*.so + @rm -f *.pyd */*.pyd */*/*.pyd @rm -f *~ */*~ */*/*~ @rm -f core */core + @rm -f Cython/*.c @rm -f Cython/Compiler/*.c @rm -f Cython/Plex/*.c @rm -f Cython/Tempita/*.c @@ -39,5 +70,38 @@ test: testclean ${PYTHON} runtests.py -vv ${TESTOPTS} +checks: + ${PYTHON} runtests.py -vv --no-unit --no-doctest --no-file --no-pyregr --no-examples + s5: $(MAKE) -C Doc/s5 slides + +qemu-user-static: + docker run --rm --privileged hypriot/qemu-register + +wheel_manylinux: sdist $(addprefix wheel_,$(MANYLINUX_IMAGES)) +$(addprefix wheel_,$(filter-out %_x86_64, $(filter-out %_i686, $(MANYLINUX_IMAGES)))): qemu-user-static + +wheel_%: dist/$(PACKAGENAME)-$(VERSION).tar.gz + echo "Building wheels for $(PACKAGENAME) $(VERSION)" + mkdir -p wheelhouse_$(subst wheel_,,$@) + time docker run --rm -t \ + -v $(shell pwd):/io \ + -e CFLAGS="$(MANYLINUX_CFLAGS)" \ + -e LDFLAGS="$(MANYLINUX_LDFLAGS) -fPIC" \ + -e WHEELHOUSE=wheelhouse$(subst wheel_musllinux,,$(subst wheel_manylinux,,$@)) \ + quay.io/pypa/$(subst wheel_,,$@) \ + bash -c '\ + rm -fr /opt/python/*pypy* ; \ + for cpdir in /opt/python/*27* ; do \ + if [ -d "$$cpdir" ]; \ + then rm -fr /opt/python/*3[78912]; \ + else rm -fr /opt/python/*{27*,3[456]*}; \ + fi; break; \ + done ; \ + ls /opt/python/ ; \ + for PYBIN in /opt/python/cp*/bin; do \ + $$PYBIN/python -V; \ + { $$PYBIN/pip wheel -w /io/$$WHEELHOUSE /io/$< & } ; \ + done; wait; \ + for whl in /io/$$WHEELHOUSE/$(PACKAGENAME)-$(VERSION)-*-linux_*.whl; do auditwheel repair $$whl -w /io/$$WHEELHOUSE; done' diff -Nru cython-0.20.1+1~201611251650-6686/MANIFEST.in cython-0.20.1+1~202203241016-9537/MANIFEST.in --- cython-0.20.1+1~201611251650-6686/MANIFEST.in 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/MANIFEST.in 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,11 @@ -include MANIFEST.in README.txt INSTALL.txt ToDo.txt USAGE.txt CHANGES.rst -include COPYING.txt LICENSE.txt 2to3-fixers.txt Makefile +include MANIFEST.in README.rst INSTALL.txt ToDo.txt USAGE.txt CHANGES.rst +include COPYING.txt LICENSE.txt Makefile include .gitrev include pylintrc include setup.py include setupegg.py include bin/* +include *requirements*.txt include cython.py cythonize.py cygdb.py recursive-include Cython *.pyx *.pxd include Cython/Parser/Grammar Cython/Parser/__init__.py diff -Nru cython-0.20.1+1~201611251650-6686/pylintrc cython-0.20.1+1~202203241016-9537/pylintrc --- cython-0.20.1+1~201611251650-6686/pylintrc 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pylintrc 2022-03-24 10:16:46.000000000 +0000 @@ -10,7 +10,7 @@ # Profiled execution. profile=no -# Add files or directories to the blacklist. They should be base names, not +# Add files or directories to the ignorelist. They should be base names, not # paths. ignore=.git,.gitmarker diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/pyxbuild.py cython-0.20.1+1~202203241016-9537/pyximport/pyxbuild.py --- cython-0.20.1+1~201611251650-6686/pyximport/pyxbuild.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/pyxbuild.py 2022-03-24 10:16:46.000000000 +0000 @@ -10,7 +10,7 @@ from distutils.extension import Extension from distutils.util import grok_environment_error try: - from Cython.Distutils.old_build_ext import old_build_ext as build_ext + from Cython.Distutils.build_ext import build_ext HAS_CYTHON = True except ImportError: HAS_CYTHON = False @@ -53,7 +53,10 @@ quiet = "--verbose" else: quiet = "--quiet" - args = [quiet, "build_ext"] + if build_in_temp: + args = [quiet, "build_ext", '--cython-c-in-temp'] + else: + args = [quiet, "build_ext"] if force_rebuild: args.append("--force") if inplace and package_base_dir: @@ -65,8 +68,6 @@ elif 'set_initial_path' not in ext.cython_directives: ext.cython_directives['set_initial_path'] = 'SOURCEFILE' - if HAS_CYTHON and build_in_temp: - args.append("--pyrex-c-in-temp") sargs = setup_args.copy() sargs.update({ "script_name": None, @@ -103,7 +104,7 @@ so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path - # when --inplace ; see http://bugs.python.org/issue5977 + # when --inplace ; see https://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) @@ -119,9 +120,9 @@ while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, - basename + '.reload%s'%count) + basename + '.reload%s' % count) try: - import shutil # late import / reload_support is: debugging + import shutil # late import / reload_support is: debugging try: # Try to unlink first --- if the .so file # is mmapped by another process, @@ -140,7 +141,7 @@ break else: # used up all 100 slots - raise ImportError("reload count for %s reached maximum"%org_path) + raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path]=(timestamp, so_path, count) return so_path except KeyboardInterrupt: @@ -157,4 +158,3 @@ if __name__=="__main__": pyx_to_dll("dummy.pyx") from . import test - diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/pyximport.py cython-0.20.1+1~202203241016-9537/pyximport/pyximport.py --- cython-0.20.1+1~201611251650-6686/pyximport/pyximport.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/pyximport.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,10 @@ """ -Import hooks; when installed with the install() function, these hooks +Import hooks; when installed with the install() function, these hooks allow importing .pyx files as if they were Python modules. If you want the hook installed every time you run Python you can add it to your Python version by adding these lines to -sitecustomize.py (which you can create from scratch in site-packages +sitecustomize.py (which you can create from scratch in site-packages if it doesn't exist there or somewhere else on your python path):: import pyximport @@ -47,15 +47,14 @@ This code is based on the Py2.3+ import protocol as described in PEP 302. """ -import sys -import os import glob import imp +import os +import sys +from zipimport import zipimporter, ZipImportError mod_name = "pyximport" -assert sys.hexversion >= 0x2030000, "need Python 2.3 or later" - PYX_EXT = ".pyx" PYXDEP_EXT = ".pyxdep" PYXBLD_EXT = ".pyxbld" @@ -79,11 +78,11 @@ # Performance problem: for every PYX file that is imported, we will -# invoke the whole distutils infrastructure even if the module is -# already built. It might be more efficient to only do it when the +# invoke the whole distutils infrastructure even if the module is +# already built. It might be more efficient to only do it when the # mod time of the .pyx is newer than the mod time of the .so but # the question is how to get distutils to tell me the name of the .so -# before it builds it. Maybe it is easy...but maybe the peformance +# before it builds it. Maybe it is easy...but maybe the performance # issue isn't real. def _load_pyrex(name, filename): "Load a pyrex file given a name and filename." @@ -94,7 +93,7 @@ # import hashlib # except ImportError: # import md5 as hashlib -# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() +# extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() # modname = modname + extra extension_mod,setup_args = handle_special_build(modname, pyxfilename) if not extension_mod: @@ -113,12 +112,13 @@ special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT ext = None setup_args={} - if os.path.exists(special_build): + if os.path.exists(special_build): # globls = {} # locs = {} # execfile(special_build, globls, locs) # ext = locs["make_ext"](modname, pyxfilename) - mod = imp.load_source("XXXX", special_build, open(special_build)) + with open(special_build) as fid: + mod = imp.load_source("XXXX", special_build, fid) make_ext = getattr(mod,'make_ext',None) if make_ext: ext = make_ext(modname, pyxfilename) @@ -126,11 +126,11 @@ make_setup_args = getattr(mod, 'make_setup_args',None) if make_setup_args: setup_args = make_setup_args() - assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" + assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" % special_build) - assert set or setup_args, ("neither make_ext nor make_setup_args %s" + assert set or setup_args, ("neither make_ext nor make_setup_args %s" % special_build) - ext.sources = [os.path.join(os.path.dirname(special_build), source) + ext.sources = [os.path.join(os.path.dirname(special_build), source) for source in ext.sources] return ext, setup_args @@ -142,10 +142,11 @@ # by default let distutils decide whether to rebuild on its own # (it has a better idea of what the output file will be) - # but we know more about dependencies so force a rebuild if + # but we know more about dependencies so force a rebuild if # some of the dependencies are newer than the pyxfile. if os.path.exists(dependfile): - depends = open(dependfile).readlines() + with open(dependfile) as fid: + depends = fid.readlines() depends = [depend.strip() for depend in depends] # gather dependencies in the "files" variable @@ -153,7 +154,7 @@ files = [dependfile] for depend in depends: fullpath = os.path.join(os.path.dirname(dependfile), - depend) + depend) files.extend(glob.glob(fullpath)) # only for unit testing to see we did the right thing @@ -191,8 +192,8 @@ inplace=inplace, reload_support=pyxargs.reload_support) assert os.path.exists(so_path), "Cannot find: %s" % so_path - - junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) + + junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) junkstuff = glob.glob(junkpath) for path in junkstuff: if path != so_path: @@ -218,7 +219,8 @@ if is_package and not hasattr(mod, '__path__'): mod.__path__ = [os.path.dirname(so_path)] assert mod.__file__ == so_path, (mod.__file__, so_path) - except Exception: + except Exception as failure_exc: + _debug("Failed to load extension module: %r" % failure_exc) if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'): # try to fall back to normal import mod = imp.load_source(name, pyxfilename) @@ -249,7 +251,11 @@ def find_module(self, fullname, package_path=None): if fullname in sys.modules and not pyxargs.reload_support: - return None # only here when reload() + return None # only here when reload() + + # package_path might be a _NamespacePath. Convert that into a list... + if package_path is not None and not isinstance(package_path, list): + package_path = list(package_path) try: fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path) if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open! @@ -266,7 +272,7 @@ pyxbuild_dir=self.pyxbuild_dir, inplace=self.inplace, language_level=self.language_level) - if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! + if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! return None # find .pyx fast, when .so/.pyd exist --inplace @@ -286,36 +292,50 @@ # searching sys.path ... #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path - if '.' in fullname: # only when package_path anyway? - mod_parts = fullname.split('.') - module_name = mod_parts[-1] - else: - module_name = fullname + + mod_parts = fullname.split('.') + module_name = mod_parts[-1] pyx_module_name = module_name + self.extension + # this may work, but it returns the file content, not its path #import pkgutil #pyx_source = pkgutil.get_data(package, pyx_module_name) - if package_path: - paths = package_path - else: - paths = sys.path - join_path = os.path.join - is_file = os.path.isfile - is_abs = os.path.isabs - abspath = os.path.abspath - #is_dir = os.path.isdir - sep = os.path.sep + paths = package_path or sys.path for path in paths: + pyx_data = None if not path: path = os.getcwd() - elif not is_abs(path): - path = abspath(path) - if is_file(path+sep+pyx_module_name): - return PyxLoader(fullname, join_path(path, pyx_module_name), - pyxbuild_dir=self.pyxbuild_dir, - inplace=self.inplace, - language_level=self.language_level) + elif os.path.isfile(path): + try: + zi = zipimporter(path) + pyx_data = zi.get_data(pyx_module_name) + except (ZipImportError, IOError, OSError): + continue # Module not found. + # unzip the imported file into the build dir + # FIXME: can interfere with later imports if build dir is in sys.path and comes before zip file + path = self.pyxbuild_dir + elif not os.path.isabs(path): + path = os.path.abspath(path) + + pyx_module_path = os.path.join(path, pyx_module_name) + if pyx_data is not None: + if not os.path.exists(path): + try: + os.makedirs(path) + except OSError: + # concurrency issue? + if not os.path.exists(path): + raise + with open(pyx_module_path, "wb") as f: + f.write(pyx_data) + elif not os.path.isfile(pyx_module_path): + continue # Module not found. + + return PyxLoader(fullname, pyx_module_path, + pyxbuild_dir=self.pyxbuild_dir, + inplace=self.inplace, + language_level=self.language_level) # not found, normal package, not a .pyx file, none of our business _debug("%s not found" % fullname) @@ -333,12 +353,13 @@ language_level=language_level) self.uncompilable_modules = {} self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild', - 'distutils.extension', 'distutils.sysconfig'] + 'distutils'] + self.blocked_packages = ['Cython.', 'distutils.'] def find_module(self, fullname, package_path=None): if fullname in sys.modules: return None - if fullname.startswith('Cython.'): + if any([fullname.startswith(pkg) for pkg in self.blocked_packages]): return None if fullname in self.blocked_modules: # prevent infinite recursion @@ -472,55 +493,60 @@ setup_args=None, reload_support=False, load_py_module_on_import_failure=False, inplace=False, language_level=None): - """Main entry point. Call this to install the .pyx import hook in + """ Main entry point for pyxinstall. + + Call this to install the ``.pyx`` import hook in your meta-path for a single Python process. If you want it to be - installed whenever you use Python, add it to your sitecustomize + installed whenever you use Python, add it to your ``sitecustomize`` (as described above). - You can pass ``pyimport=True`` to also install the .py import hook - in your meta-path. Note, however, that it is highly experimental, - will not work for most .py files, and will therefore only slow - down your imports. Use at your own risk. - - By default, compiled modules will end up in a ``.pyxbld`` - directory in the user's home directory. Passing a different path - as ``build_dir`` will override this. - - ``build_in_temp=False`` will produce the C files locally. Working - with complex dependencies and debugging becomes more easy. This - can principally interfere with existing files of the same name. - build_in_temp can be overriden by .pyxbld/make_setup_args() - by a dict item of 'build_in_temp' - - ``setup_args``: dict of arguments for Distribution - see - distutils.core.setup() . They are extended/overriden by those of - .pyxbld/make_setup_args() - - ``reload_support``: Enables support for dynamic - reload(), e.g. after a change in the Cython code. - Additional files .reloadNN may arise on that account, when - the previously loaded module file cannot be overwritten. - - ``load_py_module_on_import_failure``: If the compilation of a .py - file succeeds, but the subsequent import fails for some reason, - retry the import with the normal .py module instead of the - compiled module. Note that this may lead to unpredictable results - for modules that change the system state during their import, as - the second import will rerun these modifications in whatever state - the system was left after the import of the compiled module - failed. - - ``inplace``: Install the compiled module next to the source file. - - ``language_level``: The source language level to use: 2 or 3. - The default is to use the language level of the current Python - runtime for .py files and Py2 for .pyx files. + :param pyximport: If set to False, does not try to import ``.pyx`` files. + + :param pyimport: You can pass ``pyimport=True`` to also + install the ``.py`` import hook + in your meta-path. Note, however, that it is rather experimental, + will not work at all for some ``.py`` files and packages, and will + heavily slow down your imports due to search and compilation. + Use at your own risk. + + :param build_dir: By default, compiled modules will end up in a ``.pyxbld`` + directory in the user's home directory. Passing a different path + as ``build_dir`` will override this. + + :param build_in_temp: If ``False``, will produce the C files locally. Working + with complex dependencies and debugging becomes more easy. This + can principally interfere with existing files of the same name. + + :param setup_args: Dict of arguments for Distribution. + See ``distutils.core.setup()``. + + :param reload_support: Enables support for dynamic + ``reload(my_module)``, e.g. after a change in the Cython code. + Additional files ``.reloadNN`` may arise on that account, when + the previously loaded module file cannot be overwritten. + + :param load_py_module_on_import_failure: If the compilation of a ``.py`` + file succeeds, but the subsequent import fails for some reason, + retry the import with the normal ``.py`` module instead of the + compiled module. Note that this may lead to unpredictable results + for modules that change the system state during their import, as + the second import will rerun these modifications in whatever state + the system was left after the import of the compiled module + failed. + + :param inplace: Install the compiled module + (``.so`` for Linux and Mac / ``.pyd`` for Windows) + next to the source file. + + :param language_level: The source language level to use: 2 or 3. + The default is to use the language level of the current Python + runtime for .py files and Py2 for ``.pyx`` files. """ if setup_args is None: setup_args = {} if not build_dir: build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld') - + global pyxargs pyxargs = PyxArgs() #$pycheck_no pyxargs.build_dir = build_dir diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/README cython-0.20.1+1~202203241016-9537/pyximport/README --- cython-0.20.1+1~201611251650-6686/pyximport/README 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/README 1970-01-01 00:00:00.000000000 +0000 @@ -1,73 +0,0 @@ - == Pyximport == - -Download: pyx-import-1.0.tar.gz - - -Pyrex is a compiler. Therefore it is natural that people tend to go -through an edit/compile/test cycle with Pyrex modules. But my personal -opinion is that one of the deep insights in Python's implementation is -that a language can be compiled (Python modules are compiled to .pyc) -files and hide that compilation process from the end-user so that they -do not have to worry about it. Pyximport does this for Pyrex modules. -For instance if you write a Pyrex module called "foo.pyx", with -Pyximport you can import it in a regular Python module like this: - - -import pyximport; pyximport.install() -import foo - -Doing so will result in the compilation of foo.pyx (with appropriate -exceptions if it has an error in it). - -If you would always like to import pyrex files without building them -specially, you can also the first line above to your sitecustomize.py. -That will install the hook every time you run Python. Then you can use -Pyrex modules just with simple import statements. I like to test my -Pyrex modules like this: - - -python -c "import foo" - -See help(pyximport.install) to learn its options for controlling the -default behavior of "import" and "reload". - - == Dependency Handling == - -In Pyximport 1.1 it is possible to declare that your module depends on -multiple files, (likely ".h" and ".pxd" files). If your Pyrex module is -named "foo" and thus has the filename "foo.pyx" then you should make -another file in the same directory called "foo.pyxdep". The -"modname.pyxdep" file can be a list of filenames or "globs" (like -"*.pxd" or "include/*.h"). Each filename or glob must be on a separate -line. Pyximport will check the file date for each of those files before -deciding whether to rebuild the module. In order to keep track of the -fact that the dependency has been handled, Pyximport updates the -modification time of your ".pyx" source file. Future versions may do -something more sophisticated like informing distutils of the -dependencies directly. - - == Limitations == - -Pyximport does not give you any control over how your Pyrex file is -compiled. Usually the defaults are fine. You might run into problems if -you wanted to write your program in half-C, half-Pyrex and build them -into a single library. Pyximport 1.2 will probably do this. - -Pyximport does not hide the Distutils/GCC warnings and errors generated -by the import process. Arguably this will give you better feedback if -something went wrong and why. And if nothing went wrong it will give you -the warm fuzzy that pyximport really did rebuild your module as it was -supposed to. - - == For further thought and discussion == - -"setup.py install" does not modify sitecustomize.py for you. Should it? -Modifying Python's "standard interpreter" behaviour may be more than -most people expect of a package they install.. - -Pyximport puts your ".c" file beside your ".pyx" file (analogous to -".pyc" beside ".py"). But it puts the platform-specific binary in a -build directory as per normal for Distutils. If I could wave a magic -wand and get Pyrex or distutils or whoever to put the build directory I -might do it but not necessarily: having it at the top level is VERY -HELPFUL for debugging Pyrex problems. diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/README.rst cython-0.20.1+1~202203241016-9537/pyximport/README.rst --- cython-0.20.1+1~201611251650-6686/pyximport/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,71 @@ +Pyximport +========= + +Cython is a compiler. Therefore it is natural that people tend to go +through an edit/compile/test cycle with Cython modules. But my personal +opinion is that one of the deep insights in Python's implementation is +that a language can be compiled (Python modules are compiled to .pyc) +files and hide that compilation process from the end-user so that they +do not have to worry about it. Pyximport does this for Cython modules. +For instance if you write a Cython module called ``foo.pyx``, with +Pyximport you can import it in a regular Python module like this:: + + import pyximport; pyximport.install() + import foo + +Doing so will result in the compilation of ``foo.pyx`` (with appropriate +exceptions if it has an error in it). + +If you would always like to import Cython files without building them +specially, you can also add the first line above to your sitecustomize.py. +That will install the hook every time you run Python. Then you can use +Cython modules just with simple import statements. I like to test my +Cython modules like this:: + + python -c "import foo" + +See help(pyximport.install) to learn its options for controlling the +default behavior of ``import`` and ``reload``. + +Dependency Handling +------------------- + +In Pyximport 1.1 it is possible to declare that your module depends on +multiple files, (likely ``.h`` and ``.pxd`` files). If your Cython module is +named ``foo`` and thus has the filename ``foo.pyx`` then you should make +another file in the same directory called ``foo.pyxdep``. The +``modname.pyxdep`` file can be a list of filenames or ``globs`` (like +``*.pxd`` or ``include/*.h``). Each filename or glob must be on a separate +line. Pyximport will check the file date for each of those files before +deciding whether to rebuild the module. In order to keep track of the +fact that the dependency has been handled, Pyximport updates the +modification time of your ``.pyx`` source file. Future versions may do +something more sophisticated like informing distutils of the +dependencies directly. + +Limitations +----------- +Pyximport does not give you any control over how your Cython file is +compiled. Usually the defaults are fine. You might run into problems if +you wanted to write your program in half-C, half-Cython and build them +into a single library. Pyximport 1.2 will probably do this. + +Pyximport does not hide the Distutils/GCC warnings and errors generated +by the import process. Arguably this will give you better feedback if +something went wrong and why. And if nothing went wrong it will give you +the warm fuzzy that pyximport really did rebuild your module as it was +supposed to. + +For further thought and discussion +---------------------------------- + +``setup.py install`` does not modify ``sitecustomize.py`` for you. Should it? +Modifying Python's "standard interpreter" behaviour may be more than +most people expect of a package they install.. + +Pyximport puts your ``.c`` file beside your ``.pyx`` file (analogous to +``.pyc`` beside ``.py``). But it puts the platform-specific binary in a +build directory as per normal for Distutils. If I could wave a magic +wand and get Cython or distutils or whoever to put the build directory I +might do it but not necessarily: having it at the top level is VERY +HELPFUL for debugging Cython problems. diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/test/test_pyximport.py cython-0.20.1+1~202203241016-9537/pyximport/test/test_pyximport.py --- cython-0.20.1+1~201611251650-6686/pyximport/test/test_pyximport.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/test/test_pyximport.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,19 @@ from __future__ import absolute_import, print_function -from pyximport import pyximport; pyximport.install(reload_support=True) +from pyximport import pyximport +pyximport.install(reload_support=True) -import os, sys -import time, shutil +import os +import shutil +import sys import tempfile +import time +from zipfile import ZipFile + +try: + from __builtin__ import reload +except ImportError: + from importlib import reload def make_tempdir(): @@ -27,39 +36,42 @@ print("You may want to delete this yourself when you get a chance.") -def test(): +def test_with_reload(): pyximport._test_files = [] tempdir = make_tempdir() sys.path.append(tempdir) filename = os.path.join(tempdir, "dummy.pyx") - open(filename, "w").write("print 'Hello world from the Pyrex install hook'") + with open(filename, "w") as fid: + fid.write("print 'Hello world from the Pyrex install hook'") import dummy reload(dummy) depend_filename = os.path.join(tempdir, "dummy.pyxdep") - depend_file = open(depend_filename, "w") - depend_file.write("*.txt\nfoo.bar") - depend_file.close() + with open(depend_filename, "w") as depend_file: + depend_file.write("*.txt\nfoo.bar") build_filename = os.path.join(tempdir, "dummy.pyxbld") - build_file = open(build_filename, "w") - build_file.write(""" + with open(build_filename, "w") as build_file: + build_file.write(""" from distutils.extension import Extension def make_ext(name, filename): - return Extension(name=name, sources=[filename]) + return Extension(name=name, sources=[filename]) """) - build_file.close() - open(os.path.join(tempdir, "foo.bar"), "w").write(" ") - open(os.path.join(tempdir, "1.txt"), "w").write(" ") - open(os.path.join(tempdir, "abc.txt"), "w").write(" ") + with open(os.path.join(tempdir, "foo.bar"), "w") as fid: + fid.write(" ") + with open(os.path.join(tempdir, "1.txt"), "w") as fid: + fid.write(" ") + with open(os.path.join(tempdir, "abc.txt"), "w") as fid: + fid.write(" ") reload(dummy) assert len(pyximport._test_files)==1, pyximport._test_files reload(dummy) - time.sleep(1) # sleep a second to get safer mtimes - open(os.path.join(tempdir, "abc.txt"), "w").write(" ") - print("Here goes the reolad") + time.sleep(1) # sleep a second to get safer mtimes + with open(os.path.join(tempdir, "abc.txt"), "w") as fid: + fid.write(" ") + print("Here goes the reload") reload(dummy) assert len(pyximport._test_files) == 1, pyximport._test_files @@ -68,5 +80,40 @@ remove_tempdir(tempdir) -if __name__=="__main__": - test() +def test_zip(): + try: + import test_zip_module + except ImportError: + pass + else: + assert False, "test_zip_module already exists" + + fd, zip_path = tempfile.mkstemp(suffix=".zip") + os.close(fd) + try: + with ZipFile(zip_path, "w") as zf: + zf.writestr("test_zip_module.pyx", b"x = 42") + + sys.path.insert(0, zip_path) + import test_zip_module + assert test_zip_module.x == 42 + finally: + if zip_path in sys.path: + sys.path.remove(zip_path) + os.remove(zip_path) + + +def test_zip_nonexisting(): + sys.path.append("nonexisting_zip_module.zip") + try: + import nonexisting_zip_module + except ImportError: + pass + finally: + sys.path.remove("nonexisting_zip_module.zip") + + +if __name__== "__main__": + test_with_reload() + test_zip() + test_zip_nonexisting() diff -Nru cython-0.20.1+1~201611251650-6686/pyximport/test/test_reload.py cython-0.20.1+1~202203241016-9537/pyximport/test/test_reload.py --- cython-0.20.1+1~201611251650-6686/pyximport/test/test_reload.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/pyximport/test/test_reload.py 2022-03-24 10:16:46.000000000 +0000 @@ -18,14 +18,16 @@ tempdir = test_pyximport.make_tempdir() sys.path.append(tempdir) hello_file = os.path.join(tempdir, "hello.pyx") - open(hello_file, "w").write("x = 1; print x; before = 'before'\n") + with open(hello_file, "w") as fid: + fid.write("x = 1; print x; before = 'before'\n") import hello - assert hello.x == 1 + assert hello.x == 1 - time.sleep(1) # sleep to make sure that new "hello.pyx" has later - # timestamp than object file. + time.sleep(1) # sleep to make sure that new "hello.pyx" has later + # timestamp than object file. - open(hello_file, "w").write("x = 2; print x; after = 'after'\n") + with open(hello_file, "w") as fid: + fid.write("x = 2; print x; after = 'after'\n") reload(hello) assert hello.x == 2, "Reload should work on Python 2.3 but not 2.2" test_pyximport.remove_tempdir(tempdir) diff -Nru cython-0.20.1+1~201611251650-6686/README.rst cython-0.20.1+1~202203241016-9537/README.rst --- cython-0.20.1+1~201611251650-6686/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/README.rst 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,161 @@ +Welcome to Cython! +================== + +Cython is a language that makes writing C extensions for +Python as easy as Python itself. Cython is based on +Pyrex, but supports more cutting edge functionality and +optimizations. + +The Cython language is very close to the Python language, but Cython +additionally supports calling C functions and declaring C types on variables +and class attributes. This allows the compiler to generate very efficient C +code from Cython code. + +This makes Cython the ideal language for wrapping external C libraries, and +for fast C modules that speed up the execution of Python code. + +* Official website: https://cython.org/ +* Documentation: https://docs.cython.org/ +* Github repository: https://github.com/cython/cython +* Wiki: https://github.com/cython/cython/wiki + +You can **support the Cython project** via +`Github Sponsors `_ or +`Tidelift `_. + + +Installation: +------------- + +If you already have a C compiler, just run following command:: + + pip install Cython + +otherwise, see `the installation page `_. + + +License: +-------- + +The original Pyrex program was licensed "free of restrictions" (see below). +Cython itself is licensed under the permissive **Apache License**. + +See `LICENSE.txt `_. + + +Contributing: +------------- + +Want to contribute to the Cython project? +Here is some `help to get you started `_. + +We are currently building the next great Cython edition: +`Cython 3.0 `_. +You can help us make the life of Python 3.x users easier. + + +Differences to other Python compilers +------------------------------------- + +Started as a project in the early 2000s, Cython has outlived +`most other attempts `_ +at producing static compilers for the Python language. + +Similar projects that have a relevance today include: + +* `PyPy `_, a Python implementation with a JIT compiler. + + * Pros: JIT compilation with runtime optimisations, fully language compliant, + good integration with external C/C++ code + * Cons: non-CPython runtime, relatively large resource usage of the runtime, + limited compatibility with CPython extensions, non-obvious performance results + +* `Numba `_, a Python extension that features a + JIT compiler for a subset of the language, based on the LLVM compiler + infrastructure (probably best known for its ``clang`` C compiler). + It mostly targets numerical code that uses NumPy. + + * Pros: JIT compilation with runtime optimisations + * Cons: limited language support, relatively large runtime dependency (LLVM), + non-obvious performance results + +* `Pythran `_, a static Python-to-C++ + extension compiler for a subset of the language, mostly targeted + at numerical computation. Pythran can be (and is probably best) used + as an additional + `backend for NumPy code `_ + in Cython. + +* `mypyc `_, a static Python-to-C extension + compiler, based on the `mypy `_ static Python + analyser. Like Cython's + `pure Python mode `_, + mypyc can make use of PEP-484 type annotations to optimise code for static types. + + * Pros: good support for language and PEP-484 typing, good type inference, + reasonable performance gains + * Cons: no support for low-level optimisations and typing, + opinionated Python type interpretation, reduced Python compatibility + and introspection after compilation + +* `Nuitka `_, a static Python-to-C extension compiler. + + * Pros: highly language compliant, reasonable performance gains, + support for static application linking (similar to + `cython_freeze `_) + * Cons: no support for low-level optimisations and typing + +In comparison to the above, Cython provides + +* fast, efficient and highly compliant support for almost all + Python language features, including dynamic features and introspection +* full runtime compatibility with all still-in-use and future versions + of CPython +* "generate once, compile everywhere" C code generation that allows for + reproducible performance results and testing +* C compile time adaptation to the target platform and Python version +* support for other C-API implementations, including PyPy and Pyston +* seamless integration with C/C++ code +* broad support for manual optimisation and tuning down to the C level +* a large user base with thousands of libraries, packages and tools +* almost two decades of bug fixing and static code optimisations + + +Get the full source history: +---------------------------- + +Note that Cython used to ship the full version control repository in its source +distribution, but no longer does so due to space constraints. To get the +full source history from a downloaded source archive, make sure you have git +installed, then step into the base directory of the Cython source distribution +and type:: + + make repo + + +The following is from Pyrex: +------------------------------------------------------ +This is a development version of Pyrex, a language +for writing Python extension modules. + +For more info, take a look at: + +* Doc/About.html for a description of the language +* INSTALL.txt for installation instructions +* USAGE.txt for usage instructions +* Demos for usage examples + +Comments, suggestions, bug reports, etc. are most +welcome! + +Copyright stuff: Pyrex is free of restrictions. You +may use, redistribute, modify and distribute modified +versions. + +The latest version of Pyrex can be found `here `_. + +| Greg Ewing, Computer Science Dept +| University of Canterbury +| Christchurch, New Zealand + + A citizen of NewZealandCorp, a wholly-owned subsidiary of USA Inc. diff -Nru cython-0.20.1+1~201611251650-6686/README.txt cython-0.20.1+1~202203241016-9537/README.txt --- cython-0.20.1+1~201611251650-6686/README.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/README.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,68 +0,0 @@ -Welcome to Cython! -================= - -Cython (http://cython.org) is a language that makes writing C extensions for -the Python language as easy as Python itself. Cython is based on the -well-known Pyrex, but supports more cutting edge functionality and -optimizations. - -The Cython language is very close to the Python language, but Cython -additionally supports calling C functions and declaring C types on variables -and class attributes. This allows the compiler to generate very efficient C -code from Cython code. - -This makes Cython the ideal language for wrapping external C libraries, and -for fast C modules that speed up the execution of Python code. - -LICENSE: - -The original Pyrex program was licensed "free of restrictions" (see -below). Cython itself is licensed under the permissive - - Apache License - -See LICENSE.txt. - - --------------------------- - -Note that Cython used to ship the full version control repository in its source -distribution, but no longer does so due to space constraints. To get the -full source history, make sure you have git installed, then step into the -base directory of the Cython source distribution and type - - make repo - -Alternatively, check out the latest developer repository from - - https://github.com/cython/cython - - - -The following is from Pyrex: ------------------------------------------------------- -This is a development version of Pyrex, a language -for writing Python extension modules. - -For more info, see: - - Doc/About.html for a description of the language - INSTALL.txt for installation instructions - USAGE.txt for usage instructions - Demos for usage examples - -Comments, suggestions, bug reports, etc. are -welcome! - -Copyright stuff: Pyrex is free of restrictions. You -may use, redistribute, modify and distribute modified -versions. - -The latest version of Pyrex can be found here: - -http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/ - -Greg Ewing, Computer Science Dept, +--------------------------------------+ -University of Canterbury, | A citizen of NewZealandCorp, a | -Christchurch, New Zealand | wholly-owned subsidiary of USA Inc. | -greg@cosc.canterbury.ac.nz +--------------------------------------+ diff -Nru cython-0.20.1+1~201611251650-6686/runtests.py cython-0.20.1+1~202203241016-9537/runtests.py --- cython-0.20.1+1~201611251650-6686/runtests.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/runtests.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,9 +1,14 @@ #!/usr/bin/env python +from __future__ import print_function + +import atexit +import base64 import os import sys import re import gc +import heapq import locale import shutil import time @@ -14,14 +19,23 @@ import tempfile import traceback import warnings +import zlib +import glob +from contextlib import contextmanager +from collections import defaultdict try: import platform IS_PYPY = platform.python_implementation() == 'PyPy' IS_CPYTHON = platform.python_implementation() == 'CPython' + IS_GRAAL = platform.python_implementation() == 'GraalVM' except (ImportError, AttributeError): IS_CPYTHON = True IS_PYPY = False + IS_GRAAL = False + +IS_PY2 = sys.version_info[0] < 3 +CAN_SYMLINK = sys.platform != 'win32' and hasattr(os, 'symlink') from io import open as io_open try: @@ -40,24 +54,15 @@ threading = None try: - from collections import defaultdict + from unittest import SkipTest except ImportError: - class defaultdict(object): - def __init__(self, default_factory=lambda : None): - self._dict = {} - self.default_factory = default_factory - def __getitem__(self, key): - if key not in self._dict: - self._dict[key] = self.default_factory() - return self._dict[key] - def __setitem__(self, key, value): - self._dict[key] = value - def __contains__(self, key): - return key in self._dict - def __repr__(self): - return repr(self._dict) - def __nonzero__(self): - return bool(self._dict) + class SkipTest(Exception): # don't raise, only provided to allow except-ing it! + pass + def skip_test(reason): + sys.stderr.write("Skipping test: %s\n" % reason) +else: + def skip_test(reason): + raise SkipTest(reason) try: basestring @@ -65,44 +70,67 @@ basestring = str WITH_CYTHON = True -CY3_DIR = None from distutils.command.build_ext import build_ext as _build_ext from distutils import sysconfig +_to_clean = [] + +@atexit.register +def _cleanup_files(): + """ + This is only used on Cygwin to clean up shared libraries that are unsafe + to delete while the test suite is running. + """ + + for filename in _to_clean: + if os.path.isdir(filename): + shutil.rmtree(filename, ignore_errors=True) + else: + try: + os.remove(filename) + except OSError: + pass def get_distutils_distro(_cache=[]): if _cache: return _cache[0] - # late import to accomodate for setuptools override + # late import to accommodate for setuptools override from distutils.dist import Distribution distutils_distro = Distribution() if sys.platform == 'win32': - # TODO: Figure out why this hackery (see http://thread.gmane.org/gmane.comp.python.cython.devel/8280/). + # TODO: Figure out why this hackery (see https://thread.gmane.org/gmane.comp.python.cython.devel/8280/). config_files = distutils_distro.find_config_files() - try: config_files.remove('setup.cfg') - except ValueError: pass + try: + config_files.remove('setup.cfg') + except ValueError: + pass distutils_distro.parse_config_files(config_files) cfgfiles = distutils_distro.find_config_files() - try: cfgfiles.remove('setup.cfg') - except ValueError: pass + try: + cfgfiles.remove('setup.cfg') + except ValueError: + pass distutils_distro.parse_config_files(cfgfiles) _cache.append(distutils_distro) return distutils_distro EXT_DEP_MODULES = { - 'tag:numpy': 'numpy', + 'tag:numpy': 'numpy', + 'tag:pythran': 'pythran', + 'tag:setuptools': 'setuptools.sandbox', 'tag:asyncio': 'asyncio', 'tag:pstats': 'pstats', 'tag:posix': 'posix', 'tag:array': 'array', 'tag:coverage': 'Cython.Coverage', 'Coverage': 'Cython.Coverage', - 'tag:ipython': 'IPython', - 'tag:jedi': 'jedi', + 'tag:ipython': 'IPython.testing.globalipapp', + 'tag:jedi': 'jedi_BROKEN_AND_DISABLED', + 'tag:test.support': 'test.support', # support module for CPython unit tests } def patch_inspect_isfunction(): @@ -193,25 +221,63 @@ return '\n'.join(output) + +def exclude_extension_in_pyver(*versions): + def check(ext): + return EXCLUDE_EXT if sys.version_info[:2] in versions else ext + return check + + +def exclude_extension_on_platform(*platforms): + def check(ext): + return EXCLUDE_EXT if sys.platform in platforms else ext + return check + + def update_linetrace_extension(ext): ext.define_macros.append(('CYTHON_TRACE', 1)) return ext -def update_numpy_extension(ext): + +def update_numpy_extension(ext, set_api17_macro=True): import numpy from numpy.distutils.misc_util import get_info ext.include_dirs.append(numpy.get_include()) + if set_api17_macro and getattr(numpy, '__version__', '') not in ('1.19.0', '1.19.1'): + ext.define_macros.append(('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')) + # We need the npymath library for numpy.math. # This is typically a static-only library. for attr, value in get_info('npymath').items(): getattr(ext, attr).extend(value) + +def update_gdb_extension(ext, _has_gdb=[None]): + # We should probably also check for Python support. + if not include_debugger: + _has_gdb[0] = False + if _has_gdb[0] is None: + try: + subprocess.check_call(["gdb", "--version"]) + except (IOError, subprocess.CalledProcessError): + _has_gdb[0] = False + else: + _has_gdb[0] = True + if not _has_gdb[0]: + return EXCLUDE_EXT + return ext + + def update_openmp_extension(ext): ext.openmp = True language = ext.language + if sys.platform == 'win32' and sys.version_info[:2] == (3,4): + # OpenMP tests fail in appveyor in Py3.4 -> just ignore them, EoL of Py3.4 is early 2019... + return EXCLUDE_EXT + if language == 'cpp': flags = OPENMP_CPP_COMPILER_FLAGS else: @@ -219,7 +285,6 @@ if flags: compile_flags, link_flags = flags - ext.extra_compile_args.extend(compile_flags.split()) ext.extra_link_args.extend(link_flags.split()) return ext @@ -228,22 +293,82 @@ return EXCLUDE_EXT -def get_openmp_compiler_flags(language): + +def update_cpp11_extension(ext): """ - As of gcc 4.2, it supports OpenMP 2.5. Gcc 4.4 implements 3.0. We don't - (currently) check for other compilers. + update cpp11 extensions that will run on versions of gcc >4.8 + """ + gcc_version = get_gcc_version(ext.language) + already_has_std = any(ca for ca in ext.extra_compile_args if "-std" in ca) + if gcc_version: + compiler_version = gcc_version.group(1) + if float(compiler_version) > 4.8 and not already_has_std: + ext.extra_compile_args.append("-std=c++11") + return ext - returns a two-tuple of (CFLAGS, LDFLAGS) to build the OpenMP extension + clang_version = get_clang_version(ext.language) + if clang_version: + if not already_has_std: + ext.extra_compile_args.append("-std=c++11") + if sys.platform == "darwin": + ext.extra_compile_args.append("-stdlib=libc++") + ext.extra_compile_args.append("-mmacosx-version-min=10.7") + return ext + + return EXCLUDE_EXT + +def update_cpp17_extension(ext): + """ + update cpp17 extensions that will run on versions of gcc >=5.0 + """ + gcc_version = get_gcc_version(ext.language) + if gcc_version: + compiler_version = gcc_version.group(1) + if sys.version_info[0] < 3: + # The Python 2.7 headers contain the 'register' modifier + # which gcc warns about in C++17 mode. + ext.extra_compile_args.append('-Wno-register') + if float(compiler_version) >= 5.0: + ext.extra_compile_args.append("-std=c++17") + return ext + + clang_version = get_clang_version(ext.language) + if clang_version: + ext.extra_compile_args.append("-std=c++17") + if sys.version_info[0] < 3: + # The Python 2.7 headers contain the 'register' modifier + # which clang warns about in C++17 mode. + ext.extra_compile_args.append('-Wno-register') + if sys.platform == "darwin": + ext.extra_compile_args.append("-stdlib=libc++") + ext.extra_compile_args.append("-mmacosx-version-min=10.13") + return ext + + return EXCLUDE_EXT + +def require_gcc(version): + def check(ext): + gcc_version = get_gcc_version(ext.language) + if gcc_version: + if float(gcc_version.group(1)) >= float(version): + return ext + return EXCLUDE_EXT + return check + +def get_cc_version(language): + """ + finds gcc version using Popen """ if language == 'cpp': cc = sysconfig.get_config_var('CXX') else: cc = sysconfig.get_config_var('CC') + if not cc: + from distutils import ccompiler + cc = ccompiler.get_default_compiler() if not cc: - if sys.platform == 'win32': - return '/openmp', '' - return None + return '' # For some reason, cc can be e.g. 'gcc -pthread' cc = cc.split()[0] @@ -251,30 +376,50 @@ # Force english output env = os.environ.copy() env['LC_MESSAGES'] = 'C' - - matcher = re.compile(r"gcc version (\d+\.\d+)").search try: p = subprocess.Popen([cc, "-v"], stderr=subprocess.PIPE, env=env) - except EnvironmentError: - # Be compatible with Python 3 + except EnvironmentError as exc: warnings.warn("Unable to find the %s compiler: %s: %s" % - (language, os.strerror(sys.exc_info()[1].errno), cc)) - return None + (language, os.strerror(exc.errno), cc)) + return '' _, output = p.communicate() + return output.decode(locale.getpreferredencoding() or 'ASCII', 'replace') - output = output.decode(locale.getpreferredencoding() or 'ASCII', 'replace') - gcc_version = matcher(output) +def get_gcc_version(language): + matcher = re.compile(r"gcc version (\d+\.\d+)").search + return matcher(get_cc_version(language)) + + +def get_clang_version(language): + matcher = re.compile(r"clang(?:-|\s+version\s+)(\d+\.\d+)").search + return matcher(get_cc_version(language)) + + +def get_openmp_compiler_flags(language): + """ + As of gcc 4.2, it supports OpenMP 2.5. Gcc 4.4 implements 3.0. We don't + (currently) check for other compilers. + + returns a two-tuple of (CFLAGS, LDFLAGS) to build the OpenMP extension + """ + gcc_version = get_gcc_version(language) + if not gcc_version: - return None # not gcc - FIXME: do something about other compilers + if sys.platform == 'win32': + return '/openmp', '' + else: + return None # not gcc - FIXME: do something about other compilers # gcc defines "__int128_t", assume that at least all 64 bit architectures have it global COMPILER_HAS_INT128 COMPILER_HAS_INT128 = getattr(sys, 'maxsize', getattr(sys, 'maxint', 0)) > 2**60 compiler_version = gcc_version.group(1) - if compiler_version and compiler_version.split('.') >= ['4', '2']: - return '-fopenmp', '-fopenmp' + if compiler_version: + compiler_version = [int(num) for num in compiler_version.split('.')] + if compiler_version >= [4, 2]: + return '-fopenmp', '-fopenmp' try: locale.setlocale(locale.LC_ALL, '') @@ -292,50 +437,67 @@ EXT_EXTRAS = { 'tag:numpy' : update_numpy_extension, 'tag:openmp': update_openmp_extension, + 'tag:gdb': update_gdb_extension, + 'tag:cpp11': update_cpp11_extension, + 'tag:cpp17': update_cpp17_extension, 'tag:trace' : update_linetrace_extension, + 'tag:bytesformat': exclude_extension_in_pyver((3, 3), (3, 4)), # no %-bytes formatting + 'tag:no-macos': exclude_extension_on_platform('darwin'), + 'tag:py3only': exclude_extension_in_pyver((2, 7)), + 'tag:cppexecpolicies': require_gcc("9.1") } -def _is_py3_before_32(excluded, version): - return version[0] >= 3 and version < (3,2) - - # TODO: use tags VER_DEP_MODULES = { # tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e. # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x - (2,7) : (operator.lt, lambda x: x in ['run.withstat_py27', # multi context with statement - 'run.yield_inside_lambda', - 'run.test_dictviews', - 'run.pyclass_special_methods', - 'run.set_literals', - ]), + # The next line should start (3,); but this is a dictionary, so # we can only have one (3,) key. Since 2.7 is supposed to be the # last 2.x release, things would have to change drastically for this # to be unsafe... (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3', 'run.test_raisefrom', + 'run.different_package_names', + 'run.unicode_imports', # encoding problems on appveyor in Py2 + 'run.reimport_failure', # reimports don't do anything in Py2 ]), (3,): (operator.ge, lambda x: x in ['run.non_future_division', 'compile.extsetslice', 'compile.extdelslice', - 'run.special_methods_T561_py2' + 'run.special_methods_T561_py2', ]), - (3,1): (_is_py3_before_32, lambda x: x in ['run.pyclass_special_methods', - ]), (3,3) : (operator.lt, lambda x: x in ['build.package_compilation', + 'build.cythonize_pep420_namespace', 'run.yield_from_py33', + 'pyximport.pyximport_namespace', + 'run.qualname', ]), (3,4): (operator.lt, lambda x: x in ['run.py34_signature', + 'run.test_unicode', # taken from Py3.7, difficult to backport + 'run.pep442_tp_finalize', ]), + (3,4,999): (operator.gt, lambda x: x in ['run.initial_file_path', + ]), (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop', + 'run.py35_asyncio_async_def', + 'run.mod__spec__', + 'run.pep526_variable_annotations', # typing module + 'run.test_exceptions', # copied from Py3.7+ + 'run.time_pxd', # _PyTime_GetSystemClock doesn't exist in 3.4 + 'run.cpython_capi_py35', + 'embedding.embedded', # From the docs, needs Py_DecodeLocale + ]), + (3,7): (operator.lt, lambda x: x in ['run.pycontextvar', + 'run.pep557_dataclasses', # dataclasses module ]), } INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ] CFLAGS = os.getenv('CFLAGS', '').split() CCACHE = os.getenv('CYTHON_RUNTESTS_CCACHE', '').split() +CDEFS = [] TEST_SUPPORT_DIR = 'testsupport' BACKENDS = ['c', 'cpp'] @@ -358,8 +520,7 @@ def parse_tags(filepath): tags = defaultdict(list) parse_tag = re.compile(r'#\s*(\w+)\s*:(.*)$').match - f = io_open(filepath, encoding='ISO-8859-1', errors='ignore') - try: + with io_open(filepath, encoding='ISO-8859-1', errors='ignore') as f: for line in f: # ignore BOM-like bytes and whitespace line = line.lstrip(UTF8_BOM_BYTES).strip() @@ -384,12 +545,10 @@ tags[tag].extend(filter(None, [value.strip() for value in values])) elif tags: break # assume all tags are in one block - finally: - f.close() return tags -list_unchanging_dir = memoize(lambda x: os.listdir(x)) +list_unchanging_dir = memoize(lambda x: os.listdir(x)) # needs lambda to set function attribute @memoize @@ -400,10 +559,23 @@ if is_data_file(filename)] +def import_module_from_file(module_name, file_path, execute=True): + import importlib.util + spec = importlib.util.spec_from_file_location(module_name, file_path) + m = importlib.util.module_from_spec(spec) + if execute: + sys.modules[module_name] = m + spec.loader.exec_module(m) + return m + + def import_ext(module_name, file_path=None): if file_path: - import imp - return imp.load_dynamic(module_name, file_path) + if sys.version_info >= (3, 5): + return import_module_from_file(module_name, file_path) + else: + import imp + return imp.load_dynamic(module_name, file_path) else: try: from importlib import invalidate_caches @@ -431,59 +603,126 @@ pass _build_ext.build_extension(self, ext) + class ErrorWriter(object): - match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match - def __init__(self): + match_error = re.compile(r'(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match + + def __init__(self, encoding=None): self.output = [] - self.write = self.output.append + self.encoding = encoding - def _collect(self, collect_errors, collect_warnings): + def write(self, value): + if self.encoding: + value = value.encode('ISO-8859-1').decode(self.encoding) + self.output.append(value) + + def _collect(self): s = ''.join(self.output) - result = [] - for line in s.split('\n'): + results = {'errors': [], 'warnings': []} + for line in s.splitlines(): match = self.match_error(line) if match: is_warning, line, column, message = match.groups() - if (is_warning and collect_warnings) or \ - (not is_warning and collect_errors): - result.append( (int(line), int(column), message.strip()) ) - result.sort() - return [ "%d:%d: %s" % values for values in result ] + results['warnings' if is_warning else 'errors'].append((int(line), int(column), message.strip())) + + return [["%d:%d: %s" % values for values in sorted(results[key])] for key in ('errors', 'warnings')] def geterrors(self): - return self._collect(True, False) + return self._collect()[0] def getwarnings(self): - return self._collect(False, True) + return self._collect()[1] def getall(self): - return self._collect(True, True) + return self._collect() + + def close(self): + pass # ignore, only to match file-like interface + + +class Stats(object): + def __init__(self, top_n=8): + self.top_n = top_n + self.test_counts = defaultdict(int) + self.test_times = defaultdict(float) + self.top_tests = defaultdict(list) + + def add_time(self, name, language, metric, t, count=1): + self.test_counts[metric] += count + self.test_times[metric] += t + top = self.top_tests[metric] + push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush + # min-heap => pop smallest/shortest until longest times remain + push(top, (t, name, language)) + + @contextmanager + def time(self, name, language, metric): + t = time.time() + yield + t = time.time() - t + self.add_time(name, language, metric, t) + + def update(self, stats): + # type: (Stats) -> None + for metric, t in stats.test_times.items(): + self.test_times[metric] += t + self.test_counts[metric] += stats.test_counts[metric] + top = self.top_tests[metric] + for entry in stats.top_tests[metric]: + push = heapq.heappushpop if len(top) >= self.top_n else heapq.heappush + push(top, entry) + + def print_stats(self, out=sys.stderr): + if not self.test_times: + return + lines = ['Times:\n'] + for metric, t in sorted(self.test_times.items(), key=operator.itemgetter(1), reverse=True): + count = self.test_counts[metric] + top = self.top_tests[metric] + lines.append("%-12s: %8.2f sec (%4d, %6.3f / run) - slowest: %s\n" % ( + metric, t, count, t / count, + ', '.join("'{2}:{1}' ({0:.2f}s)".format(*item) for item in heapq.nlargest(self.top_n, top)))) + out.write(''.join(lines)) + class TestBuilder(object): - def __init__(self, rootdir, workdir, selectors, exclude_selectors, annotate, - cleanup_workdir, cleanup_sharedlibs, cleanup_failures, - with_pyregr, cython_only, languages, test_bugs, fork, language_level, - common_utility_dir): + def __init__(self, rootdir, workdir, selectors, exclude_selectors, options, + with_pyregr, languages, test_bugs, language_level, + common_utility_dir, pythran_dir=None, + default_mode='run', stats=None, + add_embedded_test=False, add_cython_import=False, + add_cpp_locals_extra_tests=False): self.rootdir = rootdir self.workdir = workdir self.selectors = selectors self.exclude_selectors = exclude_selectors - self.annotate = annotate - self.cleanup_workdir = cleanup_workdir - self.cleanup_sharedlibs = cleanup_sharedlibs - self.cleanup_failures = cleanup_failures + self.shard_num = options.shard_num + self.annotate = options.annotate_source + self.cleanup_workdir = options.cleanup_workdir + self.cleanup_sharedlibs = options.cleanup_sharedlibs + self.cleanup_failures = options.cleanup_failures self.with_pyregr = with_pyregr - self.cython_only = cython_only + self.cython_only = options.cython_only + self.test_selector = re.compile(options.only_pattern).search if options.only_pattern else None self.languages = languages self.test_bugs = test_bugs - self.fork = fork + self.fork = options.fork self.language_level = language_level + self.test_determinism = options.test_determinism self.common_utility_dir = common_utility_dir + self.pythran_dir = pythran_dir + self.default_mode = default_mode + self.stats = stats + self.add_embedded_test = add_embedded_test + self.add_cython_import = add_cython_import + self.capture = options.capture + self.add_cpp_locals_extra_tests = add_cpp_locals_extra_tests def build_suite(self): suite = unittest.TestSuite() filenames = os.listdir(self.rootdir) filenames.sort() + # TODO: parallelise I/O with a thread pool for the different directories once we drop Py2 support for filename in filenames: path = os.path.join(self.rootdir, filename) if os.path.isdir(path) and filename != TEST_SUPPORT_DIR: @@ -493,10 +732,12 @@ continue suite.addTest( self.handle_directory(path, filename)) - if sys.platform not in ['win32']: + if (sys.platform not in ['win32'] and self.add_embedded_test + # the embedding test is currently broken in Py3.8+, except on Linux. + and (sys.version_info < (3, 8) or sys.platform != 'darwin')): # Non-Windows makefile. if [1 for selector in self.selectors if selector("embedded")] \ - and not [1 for selector in self.exclude_selectors if selector("embedded")]: + and not [1 for selector in self.exclude_selectors if selector("embedded")]: suite.addTest(unittest.makeSuite(EmbedTest)) return suite @@ -528,15 +769,20 @@ if match(fqmodule, tags)]: continue - mode = 'run' # default + mode = self.default_mode if tags['mode']: mode = tags['mode'][0] elif context == 'pyregr': mode = 'pyregr' if ext == '.srctree': + if self.cython_only: + # EndToEnd tests always execute arbitrary build and test code + continue if 'cpp' not in tags['tag'] or 'cpp' in self.languages: - suite.addTest(EndToEndTest(filepath, workdir, self.cleanup_workdir)) + suite.addTest(EndToEndTest(filepath, workdir, + self.cleanup_workdir, stats=self.stats, + capture=self.capture, shard_num=self.shard_num)) continue # Choose the test suite. @@ -549,112 +795,238 @@ test_class = CythonUnitTestCase else: test_class = CythonRunTestCase - else: + elif mode in ['compile', 'error']: test_class = CythonCompileTestCase + else: + raise KeyError('Invalid test mode: ' + mode) for test in self.build_tests(test_class, path, workdir, - module, mode == 'error', tags): + module, filepath, mode == 'error', tags): suite.addTest(test) + if mode == 'run' and ext == '.py' and not self.cython_only and not filename.startswith('test_'): # additionally test file in real Python - suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename))) + min_py_ver = [ + (int(pyver.group(1)), int(pyver.group(2))) + for pyver in map(re.compile(r'pure([0-9]+)[.]([0-9]+)').match, tags['tag']) + if pyver + ] + if not min_py_ver or any(sys.version_info >= min_ver for min_ver in min_py_ver): + suite.addTest(PureDoctestTestCase( + module, filepath, tags, stats=self.stats, shard_num=self.shard_num)) return suite - def build_tests(self, test_class, path, workdir, module, expect_errors, tags): - if 'werror' in tags['tag']: - warning_errors = True - else: - warning_errors = False + def build_tests(self, test_class, path, workdir, module, module_path, expect_errors, tags): + warning_errors = 'werror' in tags['tag'] + expect_warnings = 'warnings' in tags['tag'] + + extra_directives_list = [{}] if expect_errors: - if 'cpp' in tags['tag'] and 'cpp' in self.languages: + if skip_c(tags) and 'cpp' in self.languages: languages = ['cpp'] else: languages = self.languages[:1] else: languages = self.languages - if 'cpp' in tags['tag'] and 'c' in languages: + if 'c' in languages and skip_c(tags): languages = list(languages) languages.remove('c') - elif 'no-cpp' in tags['tag'] and 'cpp' in self.languages: + if 'cpp' in languages and 'no-cpp' in tags['tag']: languages = list(languages) languages.remove('cpp') + if (self.add_cpp_locals_extra_tests and 'cpp' in languages and + 'cpp' in tags['tag'] and not 'no-cpp-locals' in tags['tag']): + extra_directives_list.append({'cpp_locals': True}) + if not languages: + return [] + + language_levels = [2, 3] if 'all_language_levels' in tags['tag'] else [None] + + pythran_dir = self.pythran_dir + if 'pythran' in tags['tag'] and not pythran_dir and 'cpp' in languages: + import pythran.config + try: + pythran_ext = pythran.config.make_extension(python=True) + except TypeError: # old pythran version syntax + pythran_ext = pythran.config.make_extension() + pythran_dir = pythran_ext['include_dirs'][0] + + add_cython_import = self.add_cython_import and module_path.endswith('.py') preparse_list = tags.get('preparse', ['id']) - tests = [ self.build_test(test_class, path, workdir, module, tags, - language, expect_errors, warning_errors, preparse) + tests = [ self.build_test(test_class, path, workdir, module, module_path, + tags, language, language_level, + expect_errors, expect_warnings, warning_errors, preparse, + pythran_dir if language == "cpp" else None, + add_cython_import=add_cython_import, + extra_directives=extra_directives) for language in languages - for preparse in preparse_list ] + for preparse in preparse_list + for language_level in language_levels + for extra_directives in extra_directives_list + ] return tests - def build_test(self, test_class, path, workdir, module, tags, - language, expect_errors, warning_errors, preparse): + def build_test(self, test_class, path, workdir, module, module_path, tags, language, language_level, + expect_errors, expect_warnings, warning_errors, preparse, pythran_dir, add_cython_import, + extra_directives): language_workdir = os.path.join(workdir, language) if not os.path.exists(language_workdir): os.makedirs(language_workdir) workdir = os.path.join(language_workdir, module) if preparse != 'id': - workdir += '_%s' % str(preparse) - return test_class(path, workdir, module, tags, + workdir += '_%s' % (preparse,) + if language_level: + workdir += '_cy%d' % (language_level,) + if extra_directives: + workdir += ('_directives_'+ '_'.join('%s_%s' % (k, v) for k,v in extra_directives.items())) + return test_class(path, workdir, module, module_path, tags, language=language, preparse=preparse, expect_errors=expect_errors, + expect_warnings=expect_warnings, annotate=self.annotate, cleanup_workdir=self.cleanup_workdir, cleanup_sharedlibs=self.cleanup_sharedlibs, cleanup_failures=self.cleanup_failures, cython_only=self.cython_only, + test_selector=self.test_selector, + shard_num=self.shard_num, fork=self.fork, - language_level=self.language_level, + language_level=language_level or self.language_level, warning_errors=warning_errors, - common_utility_dir=self.common_utility_dir) + test_determinism=self.test_determinism, + common_utility_dir=self.common_utility_dir, + pythran_dir=pythran_dir, + stats=self.stats, + add_cython_import=add_cython_import, + ) + + +def skip_c(tags): + if 'cpp' in tags['tag']: + return True + + # We don't want to create a distutils key in the + # dictionary so we check before looping. + if 'distutils' in tags: + for option in tags['distutils']: + splitted = option.split('=') + if len(splitted) == 2: + argument, value = splitted + if argument.strip() == 'language' and value.strip() == 'c++': + return True + return False + + +def filter_stderr(stderr_bytes): + """ + Filter annoying warnings from output. + """ + if b"Command line warning D9025" in stderr_bytes: + # MSCV: cl : Command line warning D9025 : overriding '/Ox' with '/Od' + stderr_bytes = b'\n'.join( + line for line in stderr_bytes.splitlines() + if b"Command line warning D9025" not in line) + return stderr_bytes + + +def filter_test_suite(test_suite, selector): + filtered_tests = [] + for test in test_suite._tests: + if isinstance(test, unittest.TestSuite): + filter_test_suite(test, selector) + elif not selector(test.id()): + continue + filtered_tests.append(test) + test_suite._tests[:] = filtered_tests + class CythonCompileTestCase(unittest.TestCase): - def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id', - expect_errors=False, annotate=False, cleanup_workdir=True, - cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False, + def __init__(self, test_directory, workdir, module, module_path, tags, language='c', preparse='id', + expect_errors=False, expect_warnings=False, annotate=False, cleanup_workdir=True, + cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False, test_selector=None, fork=True, language_level=2, warning_errors=False, - common_utility_dir=None): + test_determinism=False, shard_num=0, + common_utility_dir=None, pythran_dir=None, stats=None, add_cython_import=False, + extra_directives=None): + if extra_directives is None: + extra_directives = {} self.test_directory = test_directory self.tags = tags self.workdir = workdir self.module = module + self.module_path = module_path self.language = language self.preparse = preparse self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse) self.expect_errors = expect_errors + self.expect_warnings = expect_warnings self.annotate = annotate self.cleanup_workdir = cleanup_workdir self.cleanup_sharedlibs = cleanup_sharedlibs self.cleanup_failures = cleanup_failures self.cython_only = cython_only + self.test_selector = test_selector + self.shard_num = shard_num self.fork = fork self.language_level = language_level self.warning_errors = warning_errors + self.test_determinism = test_determinism self.common_utility_dir = common_utility_dir + self.pythran_dir = pythran_dir + self.stats = stats + self.add_cython_import = add_cython_import + self.extra_directives = extra_directives unittest.TestCase.__init__(self) def shortDescription(self): - return "compiling (%s) %s" % (self.language, self.name) + return "[%d] compiling (%s%s%s) %s" % ( + self.shard_num, + self.language, + "/cy2" if self.language_level == 2 else "/cy3" if self.language_level == 3 else "", + "/pythran" if self.pythran_dir is not None else "", + self.description_name() + ) + + def description_name(self): + return self.name def setUp(self): from Cython.Compiler import Options self._saved_options = [ (name, getattr(Options, name)) - for name in ('warning_errors', 'clear_to_none', 'error_on_unknown_names', 'error_on_uninitialized') + for name in ( + 'warning_errors', + 'clear_to_none', + 'error_on_unknown_names', + 'error_on_uninitialized', + # 'cache_builtins', # not currently supported due to incorrect global caching + ) ] self._saved_default_directives = list(Options.get_directive_defaults().items()) Options.warning_errors = self.warning_errors if sys.version_info >= (3, 4): Options._directive_defaults['autotestdict'] = False + Options._directive_defaults.update(self.extra_directives) if not os.path.exists(self.workdir): os.makedirs(self.workdir) if self.workdir not in sys.path: sys.path.insert(0, self.workdir) + if self.add_cython_import: + with open(self.module_path, 'rb') as f: + source = f.read() + if b'cython.cimports.' in source: + from Cython.Shadow import CythonCImports + for name in set(re.findall(br"(cython\.cimports(?:\.\w+)+)", source)): + name = name.decode() + sys.modules[name] = CythonCImports(name) + def tearDown(self): from Cython.Compiler import Options for name, value in self._saved_options: @@ -670,30 +1042,58 @@ del sys.modules[self.module] except KeyError: pass + + # remove any stubs of cimported modules in pure Python mode + if self.add_cython_import: + for name in list(sys.modules): + if name.startswith('cython.cimports.'): + del sys.modules[name] + cleanup = self.cleanup_failures or self.success cleanup_c_files = WITH_CYTHON and self.cleanup_workdir and cleanup cleanup_lib_files = self.cleanup_sharedlibs and cleanup + is_cygwin = sys.platform == 'cygwin' + if os.path.exists(self.workdir): - if cleanup_c_files and cleanup_lib_files: + if cleanup_c_files and cleanup_lib_files and not is_cygwin: shutil.rmtree(self.workdir, ignore_errors=True) else: for rmfile in os.listdir(self.workdir): + ext = os.path.splitext(rmfile)[1] if not cleanup_c_files: - if (rmfile[-2:] in (".c", ".h") or - rmfile[-4:] == ".cpp" or - rmfile.endswith(".html") and rmfile.startswith(self.module)): + # Keep C, C++ files, header files, preprocessed sources + # and assembly sources (typically the .i and .s files + # are intentionally generated when -save-temps is given) + if ext in (".c", ".cpp", ".h", ".i", ".ii", ".s"): continue - if not cleanup_lib_files and (rmfile.endswith(".so") or rmfile.endswith(".dll")): + if ext == ".html" and rmfile.startswith(self.module): + continue + + is_shared_obj = ext in (".so", ".dll") + + if not cleanup_lib_files and is_shared_obj: continue + try: rmfile = os.path.join(self.workdir, rmfile) if os.path.isdir(rmfile): shutil.rmtree(rmfile, ignore_errors=True) + elif is_cygwin and is_shared_obj: + # Delete later + _to_clean.append(rmfile) else: os.remove(rmfile) except IOError: pass + if cleanup_c_files and cleanup_lib_files and is_cygwin: + # Finally, remove the work dir itself + _to_clean.append(self.workdir) + + if cleanup_c_files and os.path.exists(self.workdir + '-again'): + shutil.rmtree(self.workdir + '-again', ignore_errors=True) + + def runTest(self): self.success = False self.runCompileTest() @@ -701,8 +1101,9 @@ def runCompileTest(self): return self.compile( - self.test_directory, self.module, self.workdir, - self.test_directory, self.expect_errors, self.annotate) + self.test_directory, self.module, self.module_path, self.workdir, + self.test_directory, self.expect_errors, self.expect_warnings, self.annotate, + self.add_cython_import) def find_module_source_file(self, source_file): if not os.path.exists(source_file): @@ -722,13 +1123,12 @@ if self.preparse and self.preparse != 'id': preparse_func = globals()[self.preparse] def copy(src, dest): - open(dest, 'w').write(preparse_func(open(src).read())) + with open(src) as fin: + with open(dest, 'w') as fout: + fout.write(preparse_func(fin.read())) else: # use symlink on Unix, copy on Windows - try: - copy = os.symlink - except AttributeError: - copy = shutil.copy + copy = os.symlink if CAN_SYMLINK else shutil.copy join = os.path.join for filename in file_list: @@ -741,42 +1141,49 @@ [filename for filename in file_list if not os.path.isfile(os.path.join(workdir, filename))]) - def split_source_and_output(self, test_directory, module, workdir): - source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx') - source_and_output = io_open(source_file, 'rU', encoding='ISO-8859-1') - try: - out = io_open(os.path.join(workdir, module + os.path.splitext(source_file)[1]), - 'w', encoding='ISO-8859-1') - for line in source_and_output: - if line.startswith("_ERRORS"): - out.close() - out = ErrorWriter() - else: - out.write(line) - finally: - source_and_output.close() + def split_source_and_output(self, source_file, workdir, add_cython_import=False): + from Cython.Utils import detect_opened_file_encoding + with io_open(source_file, 'rb') as f: + # encoding is passed to ErrorWriter but not used on the source + # since it is sometimes deliberately wrong + encoding = detect_opened_file_encoding(f, default=None) + + with io_open(source_file, 'r', encoding='ISO-8859-1') as source_and_output: + error_writer = warnings_writer = None + out = io_open(os.path.join(workdir, os.path.basename(source_file)), + 'w', encoding='ISO-8859-1') + try: + for line in source_and_output: + if line.startswith(u"_ERRORS"): + out.close() + out = error_writer = ErrorWriter(encoding=encoding) + elif line.startswith(u"_WARNINGS"): + out.close() + out = warnings_writer = ErrorWriter(encoding=encoding) + else: + if add_cython_import and line.strip() and not ( + line.startswith(u'#') or line.startswith(u"from __future__ import ")): + # insert "import cython" statement after any directives or future imports + if line != u"import cython\n": + out.write(u"import cython\n") + add_cython_import = False + out.write(line) + finally: + out.close() - try: - geterrors = out.geterrors - except AttributeError: - out.close() - return [] - else: - return geterrors() + return (error_writer.geterrors() if error_writer else [], + warnings_writer.geterrors() if warnings_writer else []) - def run_cython(self, test_directory, module, targetdir, incdir, annotate, + def run_cython(self, test_directory, module, module_path, targetdir, incdir, annotate, extra_compile_options=None): include_dirs = INCLUDE_DIRS + [os.path.join(test_directory, '..', TEST_SUPPORT_DIR)] if incdir: include_dirs.append(incdir) - source = self.find_module_source_file( - os.path.join(test_directory, module + '.pyx')) - if self.preparse == 'id': - source = self.find_module_source_file( - os.path.join(test_directory, module + '.pyx')) - else: - self.copy_files(test_directory, targetdir, [module + '.pyx']) - source = os.path.join(targetdir, module + '.pyx') + + if self.preparse != 'id' and test_directory != targetdir: + file_name = os.path.basename(module_path) + self.copy_files(test_directory, targetdir, [file_name]) + module_path = os.path.join(targetdir, file_name) target = os.path.join(targetdir, self.build_target_filename(module)) if extra_compile_options is None: @@ -789,9 +1196,9 @@ try: CompilationOptions except NameError: - from Cython.Compiler.Main import CompilationOptions + from Cython.Compiler.Options import CompilationOptions from Cython.Compiler.Main import compile as cython_compile - from Cython.Compiler.Main import default_options + from Cython.Compiler.Options import default_options common_utility_include_dir = self.common_utility_dir options = CompilationOptions( @@ -801,14 +1208,14 @@ annotate = annotate, use_listing_file = False, cplus = self.language == 'cpp', + np_pythran = self.pythran_dir is not None, language_level = self.language_level, generate_pxi = False, evaluate_tree_assertions = True, common_utility_include_dir = common_utility_include_dir, **extra_compile_options ) - cython_compile(source, options=options, - full_module_name=module) + cython_compile(module_path, options=options, full_module_name=module) def run_distutils(self, test_directory, module, workdir, incdir, extra_extension_args=None): @@ -824,10 +1231,8 @@ build_extension.compiler = COMPILER ext_compile_flags = CFLAGS[:] - compiler = COMPILER or sysconfig.get_config_var('CC') + ext_compile_defines = CDEFS[:] - if self.language == 'c' and compiler == 'gcc': - ext_compile_flags.extend(['-std=c89', '-pedantic']) if build_extension.compiler == 'mingw32': ext_compile_flags.append('-Wno-format') if extra_extension_args is None: @@ -841,12 +1246,17 @@ module, sources=self.source_files(workdir, module, related_files), extra_compile_args=ext_compile_flags, + define_macros=ext_compile_defines, **extra_extension_args ) if self.language == 'cpp': # Set the language now as the fixer might need it extension.language = 'c++' + if self.extra_directives.get('cpp_locals'): + extension = update_cpp17_extension(extension) + if extension is EXCLUDE_EXT: + return if 'distutils' in self.tags: from Cython.Build.Dependencies import DistutilsInfo @@ -855,6 +1265,15 @@ with open_source_file(pyx_path) as f: DistutilsInfo(f).apply(extension) + if self.pythran_dir: + from Cython.Build.Dependencies import update_pythran_extension + update_pythran_extension(extension) + + # Compile with -DCYTHON_CLINE_IN_TRACEBACK=1 unless we have + # the "traceback" tag + if 'traceback' not in self.tags['tag']: + extension.define_macros.append(("CYTHON_CLINE_IN_TRACEBACK", 1)) + for matcher, fixer in list(EXT_EXTRAS.items()): if isinstance(matcher, str): # lazy init @@ -864,14 +1283,35 @@ if matcher(module, self.tags): newext = fixer(extension) if newext is EXCLUDE_EXT: - return + return skip_test("Test '%s' excluded due to tags '%s'" % ( + self.name, ', '.join(self.tags.get('tag', '')))) extension = newext or extension if self.language == 'cpp': extension.language = 'c++' + if IS_PY2: + workdir = str(workdir) # work around type check in distutils that disallows unicode strings + build_extension.extensions = [extension] build_extension.build_temp = workdir build_extension.build_lib = workdir - build_extension.run() + + from Cython.Utils import captured_fd, prepare_captured + from distutils.errors import CompileError + + error = None + with captured_fd(2) as get_stderr: + try: + build_extension.run() + except CompileError as exc: + error = str(exc) + stderr = get_stderr() + if stderr: + # The test module name should always be ASCII, but let's not risk encoding failures. + output = b"Compiler output for module " + module.encode('utf-8') + b":\n" + stderr + b"\n" + out = sys.stdout if sys.version_info[0] == 2 else sys.stdout.buffer + out.write(output) + if error is not None: + raise CompileError(u"%s\nCompiler output:\n%s" % (error, prepare_captured(stderr))) finally: os.chdir(cwd) @@ -893,24 +1333,46 @@ return get_ext_fullpath(module) - def compile(self, test_directory, module, workdir, incdir, - expect_errors, annotate): - expected_errors = errors = () - if expect_errors: - expected_errors = self.split_source_and_output( - test_directory, module, workdir) + def compile(self, test_directory, module, module_path, workdir, incdir, + expect_errors, expect_warnings, annotate, add_cython_import): + expected_errors = expected_warnings = errors = warnings = () + if expect_errors or expect_warnings or add_cython_import: + expected_errors, expected_warnings = self.split_source_and_output( + module_path, workdir, add_cython_import) test_directory = workdir + module_path = os.path.join(workdir, os.path.basename(module_path)) if WITH_CYTHON: old_stderr = sys.stderr try: sys.stderr = ErrorWriter() - self.run_cython(test_directory, module, workdir, incdir, annotate) - errors = sys.stderr.geterrors() + with self.stats.time(self.name, self.language, 'cython'): + self.run_cython(test_directory, module, module_path, workdir, incdir, annotate) + errors, warnings = sys.stderr.getall() finally: sys.stderr = old_stderr + if self.test_determinism and not expect_errors: + workdir2 = workdir + '-again' + os.mkdir(workdir2) + self.run_cython(test_directory, module, module_path, workdir2, incdir, annotate) + diffs = [] + for file in os.listdir(workdir2): + with open(os.path.join(workdir, file)) as fid: + txt1 = fid.read() + with open(os.path.join(workdir2, file)) as fid: + txt2 = fid.read() + if txt1 != txt2: + diffs.append(file) + os.system('diff -u %s/%s %s/%s > %s/%s.diff' % ( + workdir, file, + workdir2, file, + workdir2, file)) + if diffs: + self.fail('Nondeterministic file generation: %s' % ', '.join(diffs)) tostderr = sys.__stderr__.write + if expected_warnings or (expect_warnings and warnings): + self._match_output(expected_warnings, warnings, tostderr) if 'cerror' in self.tags['tag']: if errors: tostderr("\n=== Expected C compile error ===\n") @@ -919,22 +1381,7 @@ tostderr('\n\n') raise RuntimeError('should have generated extension code') elif errors or expected_errors: - try: - for expected, error in zip(expected_errors, errors): - self.assertEquals(expected, error) - if len(errors) < len(expected_errors): - expected_error = expected_errors[len(errors)] - self.assertEquals(expected_error, None) - elif len(errors) > len(expected_errors): - unexpected_error = errors[len(expected_errors)] - self.assertEquals(None, unexpected_error) - except AssertionError: - tostderr("\n=== Expected errors: ===\n") - tostderr('\n'.join(expected_errors)) - tostderr("\n\n=== Got errors: ===\n") - tostderr('\n'.join(errors)) - tostderr('\n\n') - raise + self._match_output(expected_errors, errors, tostderr) return None so_path = None @@ -946,7 +1393,8 @@ try: with captured_fd(1) as get_stdout: with captured_fd(2) as get_stderr: - so_path = self.run_distutils(test_directory, module, workdir, incdir) + with self.stats.time(self.name, self.language, 'compile-%s' % self.language): + so_path = self.run_distutils(test_directory, module, workdir, incdir) except Exception as exc: if ('cerror' in self.tags['tag'] and ((get_stderr and get_stderr()) or @@ -960,17 +1408,40 @@ finally: if show_output: stdout = get_stdout and get_stdout().strip() + stderr = get_stderr and filter_stderr(get_stderr()).strip() + if so_path and not stderr: + # normal success case => ignore non-error compiler output + stdout = None if stdout: - tostderr("\n=== C/C++ compiler output: ===\n") - print_bytes(stdout, end=None, file=sys.__stderr__) - stderr = get_stderr and get_stderr().strip() + print_bytes( + stdout, header_text="\n=== C/C++ compiler output: =========\n", + end=None, file=sys.__stderr__) if stderr: - tostderr("\n=== C/C++ compiler error output: ===\n") - print_bytes(stderr, end=None, file=sys.__stderr__) + print_bytes( + stderr, header_text="\n=== C/C++ compiler error output: ===\n", + end=None, file=sys.__stderr__) if stdout or stderr: - tostderr("\n==============================\n") + tostderr("\n====================================\n") return so_path + def _match_output(self, expected_output, actual_output, write): + try: + for expected, actual in zip(expected_output, actual_output): + self.assertEqual(expected, actual) + if len(actual_output) < len(expected_output): + expected = expected_output[len(actual_output)] + self.assertEqual(expected, None) + elif len(actual_output) > len(expected_output): + unexpected = actual_output[len(expected_output)] + self.assertEqual(None, unexpected) + except AssertionError: + write("\n=== Expected: ===\n") + write('\n'.join(expected_output)) + write("\n\n=== Got: ===\n") + write('\n'.join(actual_output)) + write('\n\n') + raise + class CythonRunTestCase(CythonCompileTestCase): def setUp(self): @@ -978,11 +1449,8 @@ from Cython.Compiler import Options Options.clear_to_none = False - def shortDescription(self): - if self.cython_only: - return CythonCompileTestCase.shortDescription(self) - else: - return "compiling (%s) and running %s" % (self.language, self.name) + def description_name(self): + return self.name if self.cython_only else "and running %s" % self.name def run(self, result=None): if result is None: @@ -993,7 +1461,7 @@ try: self.success = False ext_so_path = self.runCompileTest() - failures, errors = len(result.failures), len(result.errors) + failures, errors, skipped = len(result.failures), len(result.errors), len(result.skipped) if not self.cython_only and ext_so_path is not None: self.run_tests(result, ext_so_path) if failures == len(result.failures) and errors == len(result.errors): @@ -1001,6 +1469,9 @@ self.success = True finally: check_thread_termination() + except SkipTest as exc: + result.addSkip(self, str(exc)) + result.stopTest(self) except Exception: result.addError(self, sys.exc_info()) result.stopTest(self) @@ -1015,13 +1486,18 @@ def run_doctests(self, module_or_name, result, ext_so_path): def run_test(result): if isinstance(module_or_name, basestring): - module = import_ext(module_or_name, ext_so_path) + with self.stats.time(self.name, self.language, 'import'): + module = import_ext(module_or_name, ext_so_path) else: module = module_or_name tests = doctest.DocTestSuite(module) - tests.run(result) + if self.test_selector: + filter_test_suite(tests, self.test_selector) + with self.stats.time(self.name, self.language, 'run'): + tests.run(result) run_forked_test(result, run_test, self.shortDescription(), self.fork) + def run_forked_test(result, run_func, test_name, fork=True): if not fork or sys.version_info[0] >= 3 or not hasattr(os, 'fork'): run_func(result) @@ -1036,7 +1512,6 @@ child_id = os.fork() if not child_id: result_code = 0 - output = None try: try: tests = partial_result = None @@ -1056,8 +1531,9 @@ _shortDescription=test_name, module_name=None) partial_result.addError(tests, sys.exc_info()) - output = open(result_file, 'wb') - pickle.dump(partial_result.data(), output) + if partial_result is not None: + with open(result_file, 'wb') as output: + pickle.dump(partial_result.data(), output) except: traceback.print_exc() finally: @@ -1065,11 +1541,6 @@ except: pass try: sys.stdout.flush() except: pass - try: - if output is not None: - output.close() - except: - pass os._exit(result_code) try: @@ -1079,30 +1550,41 @@ # upper byte of result_code, and the signal it was # killed by in the lower byte if result_code & 255: - raise Exception("Tests in module '%s' were unexpectedly killed by signal %d"% - (module_name, result_code & 255)) + raise Exception( + "Tests in module '%s' were unexpectedly killed by signal %d, see test output for details." % ( + module_name, result_code & 255)) result_code >>= 8 if result_code in (0,1): - input = open(result_file, 'rb') try: - PartialTestResult.join_results(result, pickle.load(input)) - finally: - input.close() + with open(result_file, 'rb') as f: + PartialTestResult.join_results(result, pickle.load(f)) + except Exception: + raise Exception( + "Failed to load test result from test in module '%s' after exit status %d," + " see test output for details." % (module_name, result_code)) if result_code: - raise Exception("Tests in module '%s' exited with status %d" % - (module_name, result_code)) + raise Exception( + "Tests in module '%s' exited with status %d, see test output for details." % ( + module_name, result_code)) finally: - try: os.unlink(result_file) - except: pass + try: + os.unlink(result_file) + except: + pass + class PureDoctestTestCase(unittest.TestCase): - def __init__(self, module_name, module_path): - self.module_name = module_name + def __init__(self, module_name, module_path, tags, stats=None, shard_num=0): + self.tags = tags + self.module_name = self.name = module_name self.module_path = module_path + self.stats = stats + self.shard_num = shard_num unittest.TestCase.__init__(self, 'run') def shortDescription(self): - return "running pure doctests in %s" % self.module_name + return "[%d] running pure doctests in %s" % ( + self.shard_num, self.module_name) def run(self, result=None): if result is None: @@ -1112,10 +1594,16 @@ try: self.setUp() - import imp - m = imp.load_source(loaded_module_name, self.module_path) + with self.stats.time(self.name, 'py', 'pyimport'): + if sys.version_info >= (3, 5): + m = import_module_from_file(self.module_name, self.module_path) + else: + import imp + m = imp.load_source(loaded_module_name, self.module_path) + try: - doctest.DocTestSuite(m).run(result) + with self.stats.time(self.name, 'py', 'pyrun'): + doctest.DocTestSuite(m).run(result) finally: del m if loaded_module_name in sys.modules: @@ -1129,6 +1617,22 @@ except Exception: pass + if 'mypy' in self.tags['tag']: + try: + from mypy import api as mypy_api + except ImportError: + pass + else: + with self.stats.time(self.name, 'py', 'mypy'): + mypy_result = mypy_api.run([ + self.module_path, + '--ignore-missing-imports', + '--follow-imports', 'skip', + ]) + if mypy_result[2]: + self.fail(mypy_result[0]) + + is_private_field = re.compile('^_[^_]').match class _FakeClass(object): @@ -1138,14 +1642,11 @@ def shortDescription(self): return self._shortDescription -try: # Py2.7+ and Py3.2+ - from unittest.runner import _TextTestResult -except ImportError: - from unittest import _TextTestResult +from unittest import TextTestResult -class PartialTestResult(_TextTestResult): +class PartialTestResult(TextTestResult): def __init__(self, base_result): - _TextTestResult.__init__( + TextTestResult.__init__( self, self._StringIO(), True, base_result.dots + base_result.showAll*2) @@ -1161,17 +1662,18 @@ def data(self): self.strip_error_results(self.failures) self.strip_error_results(self.errors) - return (self.failures, self.errors, self.testsRun, + return (self.failures, self.errors, self.skipped, self.testsRun, self.stream.getvalue()) def join_results(result, data): """Static method for merging the result back into the main result object. """ - failures, errors, tests_run, output = data + failures, errors, skipped, tests_run, output = data if output: result.stream.write(output) result.errors.extend(errors) + result.skipped.extend(skipped) result.failures.extend(failures) result.testsRun += tests_run @@ -1184,11 +1686,17 @@ class CythonUnitTestCase(CythonRunTestCase): def shortDescription(self): - return "compiling (%s) tests in %s" % (self.language, self.name) + return "[%d] compiling (%s) tests in %s" % ( + self.shard_num, self.language, self.description_name()) def run_tests(self, result, ext_so_path): - module = import_ext(self.module, ext_so_path) - unittest.defaultTestLoader.loadTestsFromModule(module).run(result) + with self.stats.time(self.name, self.language, 'import'): + module = import_ext(self.module, ext_so_path) + tests = unittest.defaultTestLoader.loadTestsFromModule(module) + if self.test_selector: + filter_test_suite(tests, self.test_selector) + with self.stats.time(self.name, self.language, 'run'): + tests.run(result) class CythonPyregrTestCase(CythonRunTestCase): @@ -1219,7 +1727,8 @@ suite.addTest(cls) else: suite.addTest(unittest.makeSuite(cls)) - suite.run(result) + with self.stats.time(self.name, self.language, 'run'): + suite.run(result) def _run_doctest(self, result, module): self.run_doctests(module, result, None) @@ -1243,7 +1752,8 @@ try: try: sys.stdout.flush() # helps in case of crashes - module = import_ext(self.module, ext_so_path) + with self.stats.time(self.name, self.language, 'import'): + module = import_ext(self.module, ext_so_path) sys.stdout.flush() # helps in case of crashes if hasattr(module, 'test_main'): # help 'doctest.DocFileTest' find the module path through frame inspection @@ -1264,7 +1774,27 @@ run_forked_test(result, run_test, self.shortDescription(), self.fork) -include_debugger = IS_CPYTHON and sys.version_info[:2] > (2, 5) +class TestCodeFormat(unittest.TestCase): + + def __init__(self, cython_dir): + self.cython_dir = cython_dir + unittest.TestCase.__init__(self) + + def runTest(self): + import pycodestyle + config_file = os.path.join(self.cython_dir, "setup.cfg") + if not os.path.exists(config_file): + config_file = os.path.join(os.path.dirname(__file__), "setup.cfg") + paths = [] + for codedir in ['Cython', 'Demos', 'docs', 'pyximport', 'tests']: + paths += glob.glob(os.path.join(self.cython_dir, codedir + "/**/*.py"), recursive=True) + style = pycodestyle.StyleGuide(config_file=config_file) + print("") # Fix the first line of the report. + result = style.check_files(paths) + self.assertEqual(result.total_errors, 0, "Found code style errors.") + + +include_debugger = IS_CPYTHON def collect_unittests(path, module_prefix, suite, selectors, exclude_selectors): @@ -1314,13 +1844,13 @@ return dirname not in ("Mac", "Distutils", "Plex", "Tempita") def file_matches(filename): filename, ext = os.path.splitext(filename) - blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb', - 'TestLibCython'] + excludelist = ['libcython', 'libpython', 'test_libcython_in_gdb', + 'TestLibCython'] return (ext == '.py' and not '~' in filename and not '#' in filename and not filename.startswith('.') and not - filename in blacklist) + filename in excludelist) import doctest for dirpath, dirnames, filenames in os.walk(path): for dir in list(dirnames): @@ -1357,11 +1887,14 @@ """ cython_root = os.path.dirname(os.path.abspath(__file__)) - def __init__(self, treefile, workdir, cleanup_workdir=True): + def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None, capture=True, shard_num=0): self.name = os.path.splitext(os.path.basename(treefile))[0] self.treefile = treefile self.workdir = os.path.join(workdir, self.name) self.cleanup_workdir = cleanup_workdir + self.stats = stats + self.capture = capture + self.shard_num = shard_num cython_syspath = [self.cython_root] for path in sys.path: if path.startswith(self.cython_root) and path not in cython_syspath: @@ -1373,15 +1906,14 @@ unittest.TestCase.__init__(self) def shortDescription(self): - return "End-to-end %s" % self.name + return "[%d] End-to-end %s" % ( + self.shard_num, self.name) def setUp(self): from Cython.TestUtils import unpack_source_tree - _, self.commands = unpack_source_tree(self.treefile, self.workdir) + _, self.commands = unpack_source_tree(self.treefile, self.workdir, self.cython_root) self.old_dir = os.getcwd() os.chdir(self.workdir) - if self.workdir not in sys.path: - sys.path.insert(0, self.workdir) def tearDown(self): if self.cleanup_workdir: @@ -1402,29 +1934,40 @@ def runTest(self): self.success = False - commands = (self.commands - .replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py')) - .replace("PYTHON", sys.executable)) old_path = os.environ.get('PYTHONPATH') - os.environ['PYTHONPATH'] = self.cython_syspath + os.pathsep + (old_path or '') - try: - for command in filter(None, commands.splitlines()): - p = subprocess.Popen(command, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - shell=True) - out, err = p.communicate() - res = p.returncode - if res != 0: - print(command) - print(self._try_decode(out)) - print(self._try_decode(err)) - self.assertEqual(0, res, "non-zero exit status") - finally: - if old_path: - os.environ['PYTHONPATH'] = old_path - else: - del os.environ['PYTHONPATH'] + env = dict(os.environ) + new_path = self.cython_syspath + if old_path: + new_path = new_path + os.pathsep + self.workdir + os.pathsep + old_path + env['PYTHONPATH'] = new_path + if not env.get("PYTHONIOENCODING"): + env["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() + cmd = [] + out = [] + err = [] + for command_no, command in enumerate(self.commands, 1): + with self.stats.time('%s(%d)' % (self.name, command_no), 'c', + 'etoe-build' if 'setup.py' in command else 'etoe-run'): + if self.capture: + p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env) + _out, _err = p.communicate() + res = p.returncode + else: + p = subprocess.call(command, env=env) + _out, _err = b'', b'' + res = p + cmd.append(command) + out.append(_out) + err.append(_err) + + if res == 0 and b'REFNANNY: ' in _out: + res = -1 + if res != 0: + for c, o, e in zip(cmd, out, err): + sys.stderr.write("[%d] %s\n%s\n%s\n\n" % ( + self.shard_num, c, self._try_decode(o), self._try_decode(e))) + self.assertEqual(0, res, "non-zero exit status, last output was:\n%r\n-- stdout:%s\n-- stderr:%s\n" % ( + ' '.join(command), self._try_decode(out[-1]), self._try_decode(err[-1]))) self.success = True @@ -1450,7 +1993,6 @@ os.chdir(self.old_dir) def test_embed(self): - from distutils import sysconfig libname = sysconfig.get_config_var('LIBRARY') libdir = sysconfig.get_config_var('LIBDIR') if not os.path.isdir(libdir) or libname not in os.listdir(libdir): @@ -1460,27 +2002,60 @@ if not os.path.isdir(libdir) or libname not in os.listdir(libdir): # report the error for the original directory libdir = sysconfig.get_config_var('LIBDIR') - cython = 'cython.py' - if sys.version_info[0] >=3 and CY3_DIR: - cython = os.path.join(CY3_DIR, cython) - cython = os.path.abspath(os.path.join('..', '..', cython)) - self.assert_(os.system( - "make PYTHON='%s' CYTHON='%s' LIBDIR1='%s' test > make.output" % (sys.executable, cython, libdir)) == 0) + cython = os.path.abspath(os.path.join('..', '..', 'cython.py')) + try: - os.remove('make.output') - except OSError: - pass + subprocess.check_output([ + "make", + "PYTHON='%s'" % sys.executable, + "CYTHON='%s'" % cython, + "LIBDIR1='%s'" % libdir, + "paths", "test", + ]) + except subprocess.CalledProcessError as err: + print(err.output.decode()) + raise + self.assertTrue(True) # :) + -class MissingDependencyExcluder: +def load_listfile(filename): + # just re-use the FileListExclude implementation + fle = FileListExcluder(filename) + return list(fle.excludes) + +class MissingDependencyExcluder(object): def __init__(self, deps): # deps: { matcher func : module name } self.exclude_matchers = [] - for matcher, mod in deps.items(): + for matcher, module_name in deps.items(): try: - __import__(mod) + module = __import__(module_name) except ImportError: self.exclude_matchers.append(string_selector(matcher)) + print("Test dependency not found: '%s'" % module_name) + else: + version = self.find_dep_version(module_name, module) + print("Test dependency found: '%s' version %s" % (module_name, version)) self.tests_missing_deps = [] + + def find_dep_version(self, name, module): + try: + version = module.__version__ + except AttributeError: + stdlib_dir = os.path.dirname(shutil.__file__) + os.sep + module_path = getattr(module, '__file__', stdlib_dir) # no __file__? => builtin stdlib module + # GraalPython seems to return None for some unknown reason + if module_path and module_path.startswith(stdlib_dir): + # stdlib module + version = sys.version.partition(' ')[0] + elif '.' in name: + # incrementally look for a parent package with version + name = name.rpartition('.')[0] + return self.find_dep_version(name, __import__(name)) + else: + version = '?.?' + return version + def __call__(self, testname, tags=None): for matcher in self.exclude_matchers: if matcher(testname, tags): @@ -1488,7 +2063,8 @@ return True return False -class VersionDependencyExcluder: + +class VersionDependencyExcluder(object): def __init__(self, deps): # deps: { version : matcher func } from sys import version_info @@ -1505,8 +2081,7 @@ return False -class FileListExcluder: - +class FileListExcluder(object): def __init__(self, list_file, verbose=False): self.verbose = verbose self.excludes = {} @@ -1518,16 +2093,14 @@ self.excludes[line.split()[0]] = True def __call__(self, testname, tags=None): - exclude = (testname in self.excludes - or testname.split('.')[-1] in self.excludes) + exclude = any(string_selector(ex)(testname) for ex in self.excludes) if exclude and self.verbose: print("Excluding %s because it's listed in %s" % (testname, self._list_file)) return exclude -class TagsSelector: - +class TagsSelector(object): def __init__(self, tag, value): self.tag = tag self.value = value @@ -1538,63 +2111,43 @@ else: return self.value in tags[self.tag] -class RegExSelector: +class RegExSelector(object): def __init__(self, pattern_string): try: - self.pattern = re.compile(pattern_string, re.I|re.U) + self.regex_matches = re.compile(pattern_string, re.I|re.U).search except re.error: print('Invalid pattern: %r' % pattern_string) raise def __call__(self, testname, tags=None): - return self.pattern.search(testname) + return self.regex_matches(testname) + def string_selector(s): - ix = s.find(':') - if ix == -1: - return RegExSelector(s) + if ':' in s: + return TagsSelector(*s.split(':', 1)) else: - return TagsSelector(s[:ix], s[ix+1:]) + return RegExSelector(s) + -class ShardExcludeSelector: +class ShardExcludeSelector(object): # This is an exclude selector so it can override the (include) selectors. # It may not provide uniform distribution (in time or count), but is a # determanistic partition of the tests which is important. + + # Random seed to improve the hash distribution. + _seed = base64.b64decode(b'2ged1EtsGz/GkisJr22UcLeP6n9XIaA5Vby2wM49Wvg=') + def __init__(self, shard_num, shard_count): self.shard_num = shard_num self.shard_count = shard_count - def __call__(self, testname, tags=None): - return abs(hash(testname)) % self.shard_count != self.shard_num - - -def refactor_for_py3(distdir, cy3_dir): - # need to convert Cython sources first - import lib2to3.refactor - from distutils.util import copydir_run_2to3 - with open('2to3-fixers.txt') as f: - fixers = [line.strip() for line in f if line.strip()] - if not os.path.exists(cy3_dir): - os.makedirs(cy3_dir) - import distutils.log as dlog - dlog.set_threshold(dlog.INFO) - copydir_run_2to3(distdir, cy3_dir, fixer_names=fixers, - template = ''' - global-exclude * - graft Cython - recursive-exclude Cython * - recursive-include Cython *.py *.pyx *.pxd - recursive-include Cython/Debugger/Tests * - recursive-include Cython/Utility * - recursive-exclude pyximport test - include Tools/*.py - include pyximport/*.py - include runtests.py - include cython.py - include cythonize.py - ''') - sys.path.insert(0, cy3_dir) + def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=IS_PY2): + # Cannot use simple hash() here as shard processes might use different hash seeds. + # CRC32 is fast and simple, but might return negative values in Py2. + hashval = _hash(self._seed + testname) & 0x7fffffff if _is_py2 else _hash(self._seed + testname.encode()) + return hashval % self.shard_count != self.shard_num class PendingThreadsError(RuntimeError): @@ -1605,13 +2158,13 @@ def check_thread_termination(ignore_seen=True): if threading is None: # no threading enabled in CPython return - current = threading.currentThread() + current = threading.current_thread() blocking_threads = [] for t in threading.enumerate(): - if not t.isAlive() or t == current: + if not t.is_alive() or t == current or t.name == 'time_stamper': continue t.join(timeout=2) - if t.isAlive(): + if t.is_alive(): if not ignore_seen: blocking_threads.append(t) continue @@ -1665,6 +2218,10 @@ def main(): global DISTDIR, WITH_CYTHON + + # Set an environment variable to the top directory + os.environ['CYTHON_PROJECT_DIR'] = os.path.abspath(os.path.dirname(__file__)) + DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])) from Cython.Compiler import DebugFlags @@ -1676,13 +2233,13 @@ args.append(arg) from optparse import OptionParser - parser = OptionParser() + parser = OptionParser(usage="usage: %prog [options] [selector ...]") parser.add_option("--no-cleanup", dest="cleanup_workdir", action="store_false", default=True, help="do not delete the generated C files (allows passing --no-cython on next run)") parser.add_option("--no-cleanup-sharedlibs", dest="cleanup_sharedlibs", action="store_false", default=True, - help="do not delete the generated shared libary files (allows manual module experimentation)") + help="do not delete the generated shared library files (allows manual module experimentation)") parser.add_option("--no-cleanup-failures", dest="cleanup_failures", action="store_false", default=True, help="enable --no-cleanup and --no-cleanup-sharedlibs for failed tests only") @@ -1700,6 +2257,9 @@ parser.add_option("--no-cpp", dest="use_cpp", action="store_false", default=True, help="do not test C++ compilation backend") + parser.add_option("--no-cpp-locals", dest="use_cpp_locals", + action="store_false", default=True, + help="do not rerun select C++ tests with cpp_locals directive") parser.add_option("--no-unit", dest="unittests", action="store_false", default=True, help="do not run the unit tests") @@ -1712,6 +2272,12 @@ parser.add_option("--no-pyregr", dest="pyregr", action="store_false", default=True, help="do not run the regression tests of CPython in tests/pyregr/") + parser.add_option("--no-examples", dest="examples", + action="store_false", default=True, + help="Do not run the documentation tests in the examples directory.") + parser.add_option("--no-code-style", dest="code_style", + action="store_false", default=True, + help="Do not run the code style (PEP8) checks.") parser.add_option("--cython-only", dest="cython_only", action="store_true", default=False, help="only compile pyx to c, do not run C compiler or run the tests") @@ -1727,12 +2293,18 @@ parser.add_option("-x", "--exclude", dest="exclude", action="append", metavar="PATTERN", help="exclude tests matching the PATTERN") + parser.add_option("--listfile", dest="listfile", + action="append", + help="specify a file containing a list of tests to run") parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N", type=int, default=1, help="shard this run into several parallel runs") parser.add_option("--shard_num", dest="shard_num", metavar="K", type=int, default=-1, help="test only this single shard") + parser.add_option("--profile", dest="profile", + action="store_true", default=False, + help="enable profiling of the tests") parser.add_option("-C", "--coverage", dest="coverage", action="store_true", default=False, help="collect source coverage data for the Compiler") @@ -1754,6 +2326,8 @@ parser.add_option("-T", "--ticket", dest="tickets", action="append", help="a bug ticket number to run the respective test in 'tests/*'") + parser.add_option("-k", dest="only_pattern", + help="a regex pattern for selecting doctests and test functions in the test modules") parser.add_option("-3", dest="language_level", action="store_const", const=3, default=2, help="set language level to Python 3 (useful for running the CPython regression tests)'") @@ -1762,8 +2336,15 @@ parser.add_option("--exit-ok", dest="exit_ok", default=False, action="store_true", help="exit without error code even on test failures") + parser.add_option("--failfast", dest="failfast", default=False, + action="store_true", + help="stop on first failure or error") parser.add_option("--root-dir", dest="root_dir", default=os.path.join(DISTDIR, 'tests'), - help="working directory") + help=("Directory to look for the file based " + "tests (the ones which are deactivated with '--no-file'.")) + parser.add_option("--examples-dir", dest="examples_dir", + default=os.path.join(DISTDIR, 'docs', 'examples'), + help="Directory to look for documentation example tests") parser.add_option("--work-dir", dest="work_dir", default=os.path.join(os.getcwd(), 'TEST_TMP'), help="working directory") parser.add_option("--cython-dir", dest="cython_dir", default=os.getcwd(), @@ -1776,84 +2357,116 @@ help="deterministic generated by string") parser.add_option("--use_common_utility_dir", default=False, action="store_true") parser.add_option("--use_formal_grammar", default=False, action="store_true") + parser.add_option("--test_determinism", default=False, action="store_true", + help="test whether Cython's output is deterministic") + parser.add_option("--pythran-dir", dest="pythran_dir", default=None, + help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy") + parser.add_option("--no-capture", dest="capture", default=True, action="store_false", + help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive") + parser.add_option("--limited-api", dest="limited_api", default=False, action="store_true", + help="Compiles Cython using CPython's LIMITED_API") options, cmd_args = parser.parse_args(args) - WORKDIR = os.path.abspath(options.work_dir) - if options.with_cython and sys.version_info[0] >= 3: sys.path.insert(0, options.cython_dir) - if sys.version_info[:2] == (3, 2): - try: - # try if Cython is installed in a Py3 version - import Cython.Compiler.Main - except Exception: - # back out anything the import process loaded, then - # 2to3 the Cython sources to make them re-importable - cy_modules = [ name for name in sys.modules - if name == 'Cython' or name.startswith('Cython.') ] - for name in cy_modules: - del sys.modules[name] - # hasn't been refactored yet - do it now - global CY3_DIR - CY3_DIR = cy3_dir = os.path.join(WORKDIR, 'Cy3') - refactor_for_py3(DISTDIR, cy3_dir) - if options.watermark: - import Cython.Compiler.Version - Cython.Compiler.Version.watermark = options.watermark + # requires glob with the wildcard. + if sys.version_info < (3, 5) or cmd_args: + options.code_style = False WITH_CYTHON = options.with_cython coverage = None if options.coverage or options.coverage_xml or options.coverage_html: - if options.shard_count <= 1 and options.shard_num < 0: - if not WITH_CYTHON: - options.coverage = options.coverage_xml = options.coverage_html = False - else: - print("Enabling coverage analysis") - from coverage import coverage as _coverage - coverage = _coverage(branch=True, omit=['Test*']) - coverage.erase() - coverage.start() + if not WITH_CYTHON: + options.coverage = options.coverage_xml = options.coverage_html = False + elif options.shard_num == -1: + print("Enabling coverage analysis") + from coverage import coverage as _coverage + coverage = _coverage(branch=True) + coverage.erase() + coverage.start() if options.xml_output_dir: shutil.rmtree(options.xml_output_dir, ignore_errors=True) - if WITH_CYTHON: - global CompilationOptions, pyrex_default_options, cython_compile - from Cython.Compiler.Main import \ - CompilationOptions, \ - default_options as pyrex_default_options, \ - compile as cython_compile - from Cython.Compiler import Errors - Errors.LEVEL = 0 # show all warnings - from Cython.Compiler import Options - Options.generate_cleanup_code = 3 # complete cleanup code - from Cython.Compiler import DebugFlags - DebugFlags.debug_temp_code_comments = 1 - pyrex_default_options['formal_grammar'] = options.use_formal_grammar + if options.listfile: + for listfile in options.listfile: + cmd_args.extend(load_listfile(listfile)) + if options.capture and not options.for_debugging: + keep_alive_interval = 10 + else: + keep_alive_interval = None if options.shard_count > 1 and options.shard_num == -1: + if "PYTHONIOENCODING" not in os.environ: + # Make sure subprocesses can print() Unicode text. + os.environ["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() import multiprocessing pool = multiprocessing.Pool(options.shard_count) tasks = [(options, cmd_args, shard_num) for shard_num in range(options.shard_count)] - errors = [] - for shard_num, return_code in pool.imap_unordered(runtests_callback, tasks): - if return_code != 0: - errors.append(shard_num) - print("FAILED (%s/%s)" % (shard_num, options.shard_count)) - print("ALL DONE (%s/%s)" % (shard_num, options.shard_count)) + error_shards = [] + failure_outputs = [] + # NOTE: create process pool before time stamper thread to avoid forking issues. + total_time = time.time() + stats = Stats() + merged_pipeline_stats = defaultdict(lambda: (0, 0)) + with time_stamper_thread(interval=keep_alive_interval): + for shard_num, shard_stats, pipeline_stats, return_code, failure_output in pool.imap_unordered(runtests_callback, tasks): + if return_code != 0: + error_shards.append(shard_num) + failure_outputs.append(failure_output) + sys.stderr.write("FAILED (%s/%s)\n" % (shard_num, options.shard_count)) + sys.stderr.write("ALL DONE (%s/%s)\n" % (shard_num, options.shard_count)) + + stats.update(shard_stats) + for stage_name, (stage_time, stage_count) in pipeline_stats.items(): + old_time, old_count = merged_pipeline_stats[stage_name] + merged_pipeline_stats[stage_name] = (old_time + stage_time, old_count + stage_count) + pool.close() pool.join() - if errors: - print("Errors for shards %s" % ", ".join([str(e) for e in errors])) + + total_time = time.time() - total_time + sys.stderr.write("Sharded tests run in %d seconds (%.1f minutes)\n" % (round(total_time), total_time / 60.)) + if error_shards: + sys.stderr.write("Errors found in shards %s\n" % ", ".join([str(e) for e in error_shards])) + for failure_output in zip(error_shards, failure_outputs): + sys.stderr.write("\nErrors from shard %s:\n%s" % failure_output) return_code = 1 else: return_code = 0 else: - _, return_code = runtests(options, cmd_args, coverage) - print("ALL DONE") + with time_stamper_thread(interval=keep_alive_interval): + _, stats, merged_pipeline_stats, return_code, _ = runtests(options, cmd_args, coverage) + + if coverage: + if options.shard_count > 1 and options.shard_num == -1: + coverage.combine() + coverage.stop() + + def as_msecs(t, unit=1000000): + # pipeline times are in msecs + return t // unit + float(t % unit) / unit + + pipeline_stats = [ + (as_msecs(stage_time), as_msecs(stage_time) / stage_count, stage_count, stage_name) + for stage_name, (stage_time, stage_count) in merged_pipeline_stats.items() + ] + pipeline_stats.sort(reverse=True) + sys.stderr.write("Most expensive pipeline stages: %s\n" % ", ".join( + "%r: %.2f / %d (%.3f / run)" % (stage_name, total_stage_time, stage_count, stage_time) + for total_stage_time, stage_time, stage_count, stage_name in pipeline_stats[:10] + )) + + stats.print_stats(sys.stderr) + + if coverage: + save_coverage(coverage, options) + + sys.stderr.write("ALL DONE\n") + sys.stderr.flush() try: check_thread_termination(ignore_seen=False) @@ -1864,17 +2477,116 @@ sys.exit(return_code) +@contextmanager +def time_stamper_thread(interval=10): + """ + Print regular time stamps into the build logs to find slow tests. + @param interval: time interval in seconds + """ + if not interval or interval < 0: + # Do nothing + yield + return + + try: + _xrange = xrange + except NameError: + _xrange = range + + import threading + import datetime + from time import sleep + + interval = _xrange(interval * 4) + now = datetime.datetime.now + stop = False + + # We capture stderr in some places. + # => make sure we write to the real (original) stderr of the test runner. + stderr = os.dup(2) + def write(s): + os.write(stderr, s if type(s) is bytes else s.encode('ascii')) + + def time_stamper(): + while True: + for _ in interval: + if stop: + return + sleep(1./4) + write('\n#### %s\n' % now()) + + thread = threading.Thread(target=time_stamper, name='time_stamper') + thread.setDaemon(True) # Py2 ... + thread.start() + try: + yield + finally: + stop = True + thread.join() + os.close(stderr) + + +def configure_cython(options): + global CompilationOptions, pyrex_default_options, cython_compile + from Cython.Compiler.Options import \ + CompilationOptions, \ + default_options as pyrex_default_options + from Cython.Compiler.Options import _directive_defaults as directive_defaults + from Cython.Compiler import Errors + Errors.LEVEL = 0 # show all warnings + from Cython.Compiler import Options + Options.generate_cleanup_code = 3 # complete cleanup code + from Cython.Compiler import DebugFlags + DebugFlags.debug_temp_code_comments = 1 + pyrex_default_options['formal_grammar'] = options.use_formal_grammar + if options.profile: + directive_defaults['profile'] = True + if options.watermark: + import Cython.Compiler.Version + Cython.Compiler.Version.watermark = options.watermark + + +def save_coverage(coverage, options): + if options.coverage: + coverage.report(show_missing=0) + if options.coverage_xml: + coverage.xml_report(outfile="coverage-report.xml") + if options.coverage_html: + coverage.html_report(directory="coverage-report-html") + + def runtests_callback(args): options, cmd_args, shard_num = args options.shard_num = shard_num return runtests(options, cmd_args) + def runtests(options, cmd_args, coverage=None): + # faulthandler should be able to provide a limited traceback + # in the event of a segmentation fault. Only available on Python 3.3+ + try: + import faulthandler + except ImportError: + pass # OK - not essential + else: + faulthandler.enable() + + if sys.platform == "win32" and sys.version_info < (3, 6): + # enable Unicode console output, if possible + try: + import win_unicode_console + except ImportError: + pass + else: + win_unicode_console.enable() WITH_CYTHON = options.with_cython ROOTDIR = os.path.abspath(options.root_dir) WORKDIR = os.path.abspath(options.work_dir) + if WITH_CYTHON: + configure_cython(options) + xml_output_dir = options.xml_output_dir if options.shard_num > -1: WORKDIR = os.path.join(WORKDIR, str(options.shard_num)) @@ -1905,25 +2617,34 @@ options.cleanup_sharedlibs = False options.fork = False if WITH_CYTHON and include_debugger: - from Cython.Compiler.Main import default_options as compiler_default_options + from Cython.Compiler.Options import default_options as compiler_default_options compiler_default_options['gdb_debug'] = True compiler_default_options['output_dir'] = os.getcwd() + if IS_PYPY: + if options.with_refnanny: + sys.stderr.write("Disabling refnanny in PyPy\n") + options.with_refnanny = False + if options.with_refnanny: from pyximport.pyxbuild import pyx_to_dll libpath = pyx_to_dll(os.path.join("Cython", "Runtime", "refnanny.pyx"), build_in_temp=True, pyxbuild_dir=os.path.join(WORKDIR, "support")) sys.path.insert(0, os.path.split(libpath)[0]) - CFLAGS.append("-DCYTHON_REFNANNY=1") + CDEFS.append(('CYTHON_REFNANNY', '1')) + + if options.limited_api: + CFLAGS.append("-DCYTHON_LIMITED_API=1") + CFLAGS.append('-Wno-unused-function') if xml_output_dir and options.fork: # doesn't currently work together sys.stderr.write("Disabling forked testing to support XML test output\n") options.fork = False - if WITH_CYTHON and options.language_level == 3: - sys.stderr.write("Using Cython language level 3.\n") + if WITH_CYTHON: + sys.stderr.write("Using Cython language level %d.\n" % options.language_level) test_bugs = False if options.tickets: @@ -1955,6 +2676,7 @@ exclude_selectors.append(RegExSelector('IPython')) try: + raise ImportError("Jedi typer is currently broken, see GH#1845") import jedi if not ([0, 9] <= list(map(int, re.findall('[0-9]+', jedi.__version__ or '0')))): raise ImportError @@ -1971,14 +2693,23 @@ exclude_selectors.append(ShardExcludeSelector(options.shard_num, options.shard_count)) if not test_bugs: - exclude_selectors += [ - FileListExcluder(os.path.join(ROOTDIR, bugs_file_name), verbose=verbose_excludes) - for bugs_file_name in ['bugs.txt'] + (['pypy_bugs.txt'] if IS_PYPY else []) + - (['windows_bugs.txt'] if sys.platform == 'win32' else []) + bug_files = [ + ('bugs.txt', True), + ('pypy_bugs.txt', IS_PYPY), + ('pypy2_bugs.txt', IS_PYPY and IS_PY2), + ('pypy_crash_bugs.txt', IS_PYPY), + ('pypy_implementation_detail_bugs.txt', IS_PYPY), + ('graal_bugs.txt', IS_GRAAL), + ('limited_api_bugs.txt', options.limited_api), + ('windows_bugs.txt', sys.platform == 'win32'), + ('cygwin_bugs.txt', sys.platform == 'cygwin') ] - if sys.platform in ['win32', 'cygwin'] and sys.version_info < (2,6): - exclude_selectors += [ lambda x: x == "run.specialfloat" ] + exclude_selectors += [ + FileListExcluder(os.path.join(ROOTDIR, bugs_file_name), + verbose=verbose_excludes) + for bugs_file_name, condition in bug_files if condition + ] global COMPILER if options.compiler: @@ -2000,6 +2731,13 @@ sys.stderr.write("Backends: %s\n" % ','.join(backends)) languages = backends + if 'CI' in os.environ and sys.platform == 'darwin' and 'cpp' in languages: + bugs_file_name = 'macos_cpp_bugs.txt' + exclude_selectors += [ + FileListExcluder(os.path.join(ROOTDIR, bugs_file_name), + verbose=verbose_excludes) + ] + if options.use_common_utility_dir: common_utility_dir = os.path.join(WORKDIR, 'utility_code') if not os.path.exists(common_utility_dir): @@ -2010,6 +2748,7 @@ sys.stderr.write("\n") test_suite = unittest.TestSuite() + stats = Stats() if options.unittests: collect_unittests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors, exclude_selectors) @@ -2019,71 +2758,100 @@ if options.filetests and languages: filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors, - options.annotate_source, options.cleanup_workdir, - options.cleanup_sharedlibs, options.cleanup_failures, - options.pyregr, - options.cython_only, languages, test_bugs, - options.fork, options.language_level, - common_utility_dir) + options, options.pyregr, languages, test_bugs, + options.language_level, common_utility_dir, + options.pythran_dir, add_embedded_test=True, stats=stats, + add_cpp_locals_extra_tests=options.use_cpp_locals) test_suite.addTest(filetests.build_suite()) + if options.examples and languages: + examples_workdir = os.path.join(WORKDIR, 'examples') + for subdirectory in glob.glob(os.path.join(options.examples_dir, "*/")): + filetests = TestBuilder(subdirectory, examples_workdir, selectors, exclude_selectors, + options, options.pyregr, languages, test_bugs, + options.language_level, common_utility_dir, + options.pythran_dir, + default_mode='compile', stats=stats, add_cython_import=True) + test_suite.addTest(filetests.build_suite()) + if options.system_pyregr and languages: sys_pyregr_dir = os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test') if not os.path.isdir(sys_pyregr_dir): sys_pyregr_dir = os.path.join(os.path.dirname(sys.executable), 'Lib', 'test') # source build if os.path.isdir(sys_pyregr_dir): filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors, - options.annotate_source, options.cleanup_workdir, - options.cleanup_sharedlibs, options.cleanup_failures, - True, - options.cython_only, languages, test_bugs, - options.fork, sys.version_info[0], - common_utility_dir) + options, True, languages, test_bugs, + sys.version_info[0], common_utility_dir, stats=stats) sys.stderr.write("Including CPython regression tests in %s\n" % sys_pyregr_dir) test_suite.addTest(filetests.handle_directory(sys_pyregr_dir, 'pyregr')) + if options.code_style and options.shard_num <= 0: + try: + import pycodestyle + except ImportError: + # Hack to make the exclusion visible. + missing_dep_excluder.tests_missing_deps.append('TestCodeFormat') + else: + test_suite.addTest(TestCodeFormat(options.cython_dir)) + if xml_output_dir: from Cython.Tests.xmlrunner import XMLTestRunner if not os.path.exists(xml_output_dir): - try: os.makedirs(xml_output_dir) - except OSError: pass # concurrency issue? + try: + os.makedirs(xml_output_dir) + except OSError: + pass # concurrency issue? test_runner = XMLTestRunner(output=xml_output_dir, verbose=options.verbosity > 0) + if options.failfast: + sys.stderr.write("--failfast not supported with XML runner\n") else: - test_runner = unittest.TextTestRunner(verbosity=options.verbosity) + text_runner_options = {} + if options.failfast: + text_runner_options['failfast'] = True + test_runner = unittest.TextTestRunner(verbosity=options.verbosity, **text_runner_options) if options.pyximport_py: from pyximport import pyximport pyximport.install(pyimport=True, build_dir=os.path.join(WORKDIR, '_pyximport'), load_py_module_on_import_failure=True, inplace=True) - result = test_runner.run(test_suite) + try: + gc.set_debug(gc.DEBUG_UNCOLLECTABLE) + except AttributeError: + pass # not available on PyPy + + enable_faulthandler = False + try: + import faulthandler + except ImportError: + pass + else: + enable_faulthandler = not faulthandler.is_enabled() + if enable_faulthandler: + faulthandler.enable() + + # Run the collected tests. + try: + if options.shard_num > -1: + sys.stderr.write("Tests in shard %d/%d starting\n" % (options.shard_num, options.shard_count)) + result = test_runner.run(test_suite) + except Exception as exc: + # Make sure we print exceptions also from shards. + if options.shard_num > -1: + sys.stderr.write("Tests in shard %d/%d crashed: %s\n" % (options.shard_num, options.shard_count, exc)) + import traceback + traceback.print_exc() + raise + finally: + if enable_faulthandler: + faulthandler.disable() if common_utility_dir and options.shard_num < 0 and options.cleanup_workdir: shutil.rmtree(common_utility_dir) - if coverage is not None: - coverage.stop() - ignored_modules = set( - 'Cython.Compiler.' + name - for name in ('Version', 'DebugFlags', 'CmdLine')) | set( - 'Cython.' + name - for name in ('Debugging',)) - ignored_packages = ['Cython.Runtime', 'Cython.Tempita'] - modules = [ - module for name, module in sys.modules.items() - if module is not None and - name.startswith('Cython.') and - '.Tests' not in name and - name not in ignored_modules and - not any(name.startswith(package) for package in ignored_packages) - ] - if options.coverage: - coverage.report(modules, show_missing=0) - if options.coverage_xml: - coverage.xml_report(modules, outfile="coverage-report.xml") - if options.coverage_html: - coverage.html_report(modules, directory="coverage-report-html") + from Cython.Compiler.Pipeline import get_timings + pipeline_stats = get_timings() if missing_dep_excluder.tests_missing_deps: sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n") @@ -2094,10 +2862,27 @@ import refnanny sys.stderr.write("\n".join([repr(x) for x in refnanny.reflog])) - if options.exit_ok: - return options.shard_num, 0 + result_code = 0 if options.exit_ok else not result.wasSuccessful() + + if xml_output_dir: + failure_output = "" else: - return options.shard_num, not result.wasSuccessful() + failure_output = "".join(collect_failure_output(result)) + + return options.shard_num, stats, pipeline_stats, result_code, failure_output + + +def collect_failure_output(result): + """Extract test error/failure output from a TextTestResult.""" + failure_output = [] + for flavour, errors in (("ERROR", result.errors), ("FAIL", result.failures)): + for test, err in errors: + failure_output.append("%s\n%s: %s\n%s\n%s\n" % ( + result.separator1, + flavour, result.getDescription(test), + result.separator2, + err)) + return failure_output if __name__ == '__main__': diff -Nru cython-0.20.1+1~201611251650-6686/setup.cfg cython-0.20.1+1~202203241016-9537/setup.cfg --- cython-0.20.1+1~201611251650-6686/setup.cfg 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/setup.cfg 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,43 @@ +[flake8] +max-complexity = 10 + +[pycodestyle] +exclude = .git,build,__pycache__,venv*,TEST*,tests/run/test*.py,Cython/Debugger/libpython.py +max-line-length = 150 +format = pylint +# See https://pycodestyle.pycqa.org/en/latest/intro.html#configuration +select = + E711, E713, E714, E501, W291, E502, E703, + # indentation + E101, E111, E112, E113, E117 + E121, E125, E129, + # E114, E115, E116, E122, + # whitespace + E211, E223, E224, E227, E228, E242, E261, E273, E274, E275, + # E201, E202, E203, E211, E265 + # E303, E306, + W1, W2, W3 +#ignore = W, E +ignore = + W504, + # W504 line break after binary operator + S001, + # S001 found module formatter + E226, + # E226 missing whitespace around operator[run] + +[coverage:run] +branch = True +parallel = True +concurrency = multiprocessing,thread +include = Cython/* +source = Cython +omit = Test* + +[bdist_wheel] +universal = 1 + +[metadata] +license_files = + LICENSE.txt + COPYING.txt diff -Nru cython-0.20.1+1~201611251650-6686/setup.py cython-0.20.1+1~202203241016-9537/setup.py --- cython-0.20.1+1~201611251650-6686/setup.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/setup.py 2022-03-24 10:16:46.000000000 +0000 @@ -12,6 +12,10 @@ import platform is_cpython = platform.python_implementation() == 'CPython' +# this specifies which versions of python we support, pip >= 9 knows to skip +# versions of packages which are not compatible with the running python +PYTHON_REQUIRES = '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + if sys.platform == "darwin": # Don't create resource files on OS X tar. os.environ['COPY_EXTENDED_ATTRIBUTES_DISABLE'] = 'true' @@ -34,20 +38,11 @@ sdist_orig.run(self) add_command_class('sdist', sdist) -if sys.version_info[:2] == (3, 2): - import lib2to3.refactor - from distutils.command.build_py \ - import build_py_2to3 as build_py - # need to convert sources to Py3 on installation - with open('2to3-fixers.txt') as f: - fixers = [line.strip() for line in f if line.strip()] - build_py.fixer_names = fixers - add_command_class("build_py", build_py) - pxd_include_dirs = [ - directory for directory, dirs, files in os.walk('Cython/Includes') + directory for directory, dirs, files + in os.walk(os.path.join('Cython', 'Includes')) if '__init__.pyx' in files or '__init__.pxd' in files - or directory == 'Cython/Includes' or directory == 'Cython/Includes/Deprecated'] + or directory == os.path.join('Cython', 'Includes')] pxd_include_patterns = [ p+'/*.pxd' for p in pxd_include_dirs ] + [ @@ -59,58 +54,57 @@ 'Cython.Runtime' : ['*.pyx', '*.pxd'], 'Cython.Utility' : ['*.pyx', '*.pxd', '*.c', '*.h', '*.cpp'], 'Cython' : [ p[7:] for p in pxd_include_patterns ], - } + 'Cython.Debugger.Tests': ['codefile', 'cfuncs.c'], +} # This dict is used for passing extra arguments that are setuptools # specific to setup setuptools_extra_args = {} -# tells whether to include cygdb (the script and the Cython.Debugger package -include_debugger = sys.version_info[:2] > (2, 5) - if 'setuptools' in sys.modules: + setuptools_extra_args['python_requires'] = PYTHON_REQUIRES setuptools_extra_args['zip_safe'] = False setuptools_extra_args['entry_points'] = { 'console_scripts': [ 'cython = Cython.Compiler.Main:setuptools_main', - 'cythonize = Cython.Build.Cythonize:main' + 'cythonize = Cython.Build.Cythonize:main', + 'cygdb = Cython.Debugger.Cygdb:main', ] } scripts = [] else: if os.name == "posix": - scripts = ["bin/cython", 'bin/cythonize'] + scripts = ["bin/cython", "bin/cythonize", "bin/cygdb"] else: - scripts = ["cython.py", "cythonize.py"] - -if include_debugger: - if 'setuptools' in sys.modules: - setuptools_extra_args['entry_points']['console_scripts'].append( - 'cygdb = Cython.Debugger.Cygdb:main') - else: - if os.name == "posix": - scripts.append('bin/cygdb') - else: - scripts.append('cygdb.py') + scripts = ["cython.py", "cythonize.py", "cygdb.py"] -def compile_cython_modules(profile=False, compile_more=False, cython_with_refnanny=False): +def compile_cython_modules(profile=False, coverage=False, compile_minimal=False, compile_more=False, cython_with_refnanny=False): source_root = os.path.abspath(os.path.dirname(__file__)) compiled_modules = [ - "Cython.Plex.Scanners", "Cython.Plex.Actions", - "Cython.Compiler.Lexicon", + "Cython.Plex.Scanners", + "Cython.Compiler.FlowControl", "Cython.Compiler.Scanning", - "Cython.Compiler.Parsing", "Cython.Compiler.Visitor", - "Cython.Compiler.FlowControl", - "Cython.Compiler.Code", "Cython.Runtime.refnanny", - # "Cython.Compiler.FusedNode", - "Cython.Tempita._tempita", ] - if compile_more: + if not compile_minimal: + compiled_modules.extend([ + "Cython.Plex.Machines", + "Cython.Plex.Transitions", + "Cython.Plex.DFA", + "Cython.Compiler.FusedNode", + "Cython.Tempita._tempita", + "Cython.StringIOTree", + "Cython.Utils", + ]) + if compile_more and not compile_minimal: compiled_modules.extend([ + "Cython.Compiler.Code", + "Cython.Compiler.Lexicon", + "Cython.Compiler.Parsing", + "Cython.Compiler.Pythran", "Cython.Build.Dependencies", "Cython.Compiler.ParseTreeTransforms", "Cython.Compiler.Nodes", @@ -145,87 +139,58 @@ defines = [] if cython_with_refnanny: defines.append(('CYTHON_REFNANNY', '1')) + if coverage: + defines.append(('CYTHON_TRACE', '1')) extensions = [] for module in compiled_modules: source_file = os.path.join(source_root, *module.split('.')) - if os.path.exists(source_file + ".py"): - pyx_source_file = source_file + ".py" - else: - pyx_source_file = source_file + ".pyx" + pyx_source_file = source_file + ".py" + if not os.path.exists(pyx_source_file): + pyx_source_file += "x" # .py -> .pyx + dep_files = [] if os.path.exists(source_file + '.pxd'): dep_files.append(source_file + '.pxd') - if '.refnanny' in module: - defines_for_module = [] - else: - defines_for_module = defines + extensions.append(Extension( module, sources=[pyx_source_file], - define_macros=defines_for_module, + define_macros=defines if '.refnanny' not in module else [], depends=dep_files)) # XXX hack around setuptools quirk for '*.pyx' sources extensions[-1].sources[0] = pyx_source_file - if sys.version_info[:2] == (3, 2): - # Python 3.2: can only run Cython *after* running 2to3 - build_ext = _defer_cython_import_in_py32(source_root, profile) - else: - from Cython.Distutils import build_ext - if profile: - from Cython.Compiler.Options import directive_defaults - directive_defaults['profile'] = True - sys.stderr.write("Enabled profiling for the Cython binary modules\n") + # optimise build parallelism by starting with the largest modules + extensions.sort(key=lambda ext: os.path.getsize(ext.sources[0]), reverse=True) - # not using cythonize() here to let distutils decide whether building extensions was requested + from Cython.Distutils.build_ext import build_ext + from Cython.Compiler.Options import get_directive_defaults + get_directive_defaults().update( + language_level=2, + binding=False, + always_allow_keywords=False, + autotestdict=False, + ) + if profile: + get_directive_defaults()['profile'] = True + sys.stderr.write("Enabled profiling for the Cython binary modules\n") + if coverage: + get_directive_defaults()['linetrace'] = True + sys.stderr.write("Enabled line tracing and profiling for the Cython binary modules\n") + + # not using cythonize() directly to let distutils decide whether building extensions was requested add_command_class("build_ext", build_ext) setup_args['ext_modules'] = extensions -def _defer_cython_import_in_py32(source_root, profile=False): - # Python 3.2: can only run Cython *after* running 2to3 - # => re-import Cython inside of build_ext - from distutils.command.build_ext import build_ext as build_ext_orig - - class build_ext(build_ext_orig): - # we must keep the original modules alive to make sure - # their code keeps working when we remove them from - # sys.modules - dead_modules = [] - - def __reimport(self): - if self.dead_modules: - return - # add path where 2to3 installed the transformed sources - # and make sure Python (re-)imports them from there - already_imported = [ - module for module in sys.modules - if module == 'Cython' or module.startswith('Cython.') - ] - keep_alive = self.dead_modules.append - for module in already_imported: - keep_alive(sys.modules[module]) - del sys.modules[module] - sys.path.insert(0, os.path.join(source_root, self.build_lib)) - - def build_extensions(self): - self.__reimport() - if profile: - from Cython.Compiler.Options import directive_defaults - directive_defaults['profile'] = True - print("Enabled profiling for the Cython binary modules") - from Cython.Build.Dependencies import cythonize - self.distribution.ext_modules[:] = cythonize( - self.distribution.ext_modules) - build_ext_orig.build_extensions(self) - - return build_ext - - cython_profile = '--cython-profile' in sys.argv if cython_profile: sys.argv.remove('--cython-profile') +cython_coverage = '--cython-coverage' in sys.argv +if cython_coverage: + sys.argv.remove('--cython-coverage') + try: sys.argv.remove("--cython-compile-all") cython_compile_more = True @@ -233,6 +198,12 @@ cython_compile_more = False try: + sys.argv.remove("--cython-compile-minimal") + cython_compile_minimal = True +except ValueError: + cython_compile_minimal = False + +try: sys.argv.remove("--cython-with-refnanny") cython_with_refnanny = True except ValueError: @@ -244,15 +215,10 @@ except ValueError: compile_cython_itself = True -if compile_cython_itself and (is_cpython or cython_compile_more): - compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny) - setup_args.update(setuptools_extra_args) -from Cython import __version__ as version - -def dev_status(): +def dev_status(version): if 'b' in version or 'c' in version: # 1b1, 1beta1, 2rc1, ... return 'Development Status :: 4 - Beta' @@ -269,6 +235,8 @@ 'Cython.Compiler', 'Cython.Runtime', 'Cython.Distutils', + 'Cython.Debugger', + 'Cython.Debugger.Tests', 'Cython.Plex', 'Cython.Tests', 'Cython.Build.Tests', @@ -278,60 +246,75 @@ 'pyximport', ] -if include_debugger: - packages.append('Cython.Debugger') - packages.append('Cython.Debugger.Tests') - # it's enough to do this for Py2.5+: - setup_args['package_data']['Cython.Debugger.Tests'] = ['codefile', 'cfuncs.c'] - -setup( - name='Cython', - version=version, - url='http://cython.org/', - author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', - author_email='cython-devel@python.org', - description="The Cython compiler for writing C extensions for the Python language.", - long_description=textwrap.dedent("""\ - The Cython language makes writing C extensions for the Python language as - easy as Python itself. Cython is a source code translator based on Pyrex_, - but supports more cutting edge functionality and optimizations. - - The Cython language is a superset of the Python language (almost all Python - code is also valid Cython code), but Cython additionally supports optional - static typing to natively call C functions, operate with C++ classes and - declare fast C types on variables and class attributes. This allows the - compiler to generate very efficient C code from Cython code. - - This makes Cython the ideal language for writing glue code for external - C/C++ libraries, and for fast C modules that speed up the execution of - Python code. - - Note that for one-time builds, e.g. for CI/testing, on platforms that are not - covered by one of the wheel packages provided on PyPI, it is substantially faster - than a full source build to install an uncompiled (slower) version of Cython with:: - - pip install Cython --install-option="--no-cython-compile" - - .. _Pyrex: http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ - """), - license='Apache', - classifiers=[ - dev_status(), - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 3", - "Programming Language :: C", - "Programming Language :: Cython", - "Topic :: Software Development :: Code Generators", - "Topic :: Software Development :: Compilers", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - - scripts=scripts, - packages=packages, - py_modules=["cython"], - **setup_args -) + +def run_build(): + if compile_cython_itself and (is_cpython or cython_compile_more or cython_compile_minimal): + compile_cython_modules(cython_profile, cython_coverage, cython_compile_minimal, cython_compile_more, cython_with_refnanny) + + from Cython import __version__ as version + setup( + name='Cython', + version=version, + url='https://cython.org/', + author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', + author_email='cython-devel@python.org', + description="The Cython compiler for writing C extensions for the Python language.", + long_description=textwrap.dedent("""\ + The Cython language makes writing C extensions for the Python language as + easy as Python itself. Cython is a source code translator based on Pyrex_, + but supports more cutting edge functionality and optimizations. + + The Cython language is a superset of the Python language (almost all Python + code is also valid Cython code), but Cython additionally supports optional + static typing to natively call C functions, operate with C++ classes and + declare fast C types on variables and class attributes. This allows the + compiler to generate very efficient C code from Cython code. + + This makes Cython the ideal language for writing glue code for external + C/C++ libraries, and for fast C modules that speed up the execution of + Python code. + + Note that for one-time builds, e.g. for CI/testing, on platforms that are not + covered by one of the wheel packages provided on PyPI *and* the pure Python wheel + that we provide is not used, it is substantially faster than a full source build + to install an uncompiled (slower) version of Cython with:: + + pip install Cython --install-option="--no-cython-compile" + + .. _Pyrex: https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + """), + license='Apache', + classifiers=[ + dev_status(version), + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Programming Language :: Cython", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Compilers", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + + scripts=scripts, + packages=packages, + py_modules=["cython"], + **setup_args + ) + + +if __name__ == '__main__': + run_build() diff -Nru cython-0.20.1+1~201611251650-6686/test-requirements-27.txt cython-0.20.1+1~202203241016-9537/test-requirements-27.txt --- cython-0.20.1+1~201611251650-6686/test-requirements-27.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/test-requirements-27.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,62 @@ +attrs==20.3.0 +backports-abc==0.5 +backports.functools-lru-cache==1.6.3 +backports.shutil-get-terminal-size==1.0.0 +bleach==3.3.0 +configparser==4.0.2 +contextlib2==0.6.0.post1 +coverage==5.5 +decorator==4.4.2 +defusedxml==0.7.1 +entrypoints==0.3 +enum34==1.1.10 +functools32==3.2.3.post2 +futures==3.3.0 +importlib-metadata==2.1.1 +ipaddress==1.0.23 +ipykernel==4.10.1 +ipython==5.10.0 +ipython-genutils==0.2.0 +ipywidgets==7.6.3 +Jinja2==2.11.3 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==5.3.5 +jupyter-console==5.2.0 +jupyter-core==4.6.3 +line-profiler==3.1.0 +MarkupSafe==1.1.1 +mistune==0.8.4 +nbconvert==5.6.1 +nbformat==4.4.0 +notebook==5.7.10 +numpy==1.16.6 +packaging==20.9 +pandocfilters==1.4.3 +pathlib2==2.3.5 +pexpect==4.8.0 +pickleshare==0.7.5 +prometheus-client==0.10.0 +prompt-toolkit==1.0.18 +ptyprocess==0.7.0 +pycodestyle==2.7.0 +Pygments==2.5.2 +pyparsing==2.4.7 +pyrsistent==0.15.7 +python-dateutil==2.8.1 +pyzmq==16.0.4 +qtconsole==4.7.7 +QtPy==1.9.0 +scandir==1.10.0 +Send2Trash==1.5.0 +simplegeneric==0.8.1 +singledispatch==3.6.1 +six==1.15.0 +terminado==0.8.3 +testpath==0.4.4 +tornado==5.1.1 +traitlets==4.3.3 +wcwidth==0.2.5 +webencodings==0.5.1 +widgetsnbextension==3.5.1 +zipp==1.2.0 diff -Nru cython-0.20.1+1~201611251650-6686/test-requirements-34.txt cython-0.20.1+1~202203241016-9537/test-requirements-34.txt --- cython-0.20.1+1~201611251650-6686/test-requirements-34.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/test-requirements-34.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,3 @@ +numpy < 1.19.0 +coverage +pycodestyle diff -Nru cython-0.20.1+1~201611251650-6686/test-requirements-cpython.txt cython-0.20.1+1~202203241016-9537/test-requirements-cpython.txt --- cython-0.20.1+1~201611251650-6686/test-requirements-cpython.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/test-requirements-cpython.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,2 @@ +jupyter +line_profiler diff -Nru cython-0.20.1+1~201611251650-6686/test-requirements.txt cython-0.20.1+1~202203241016-9537/test-requirements.txt --- cython-0.20.1+1~201611251650-6686/test-requirements.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/test-requirements.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,3 @@ +numpy +coverage +pycodestyle diff -Nru cython-0.20.1+1~201611251650-6686/tests/broken/cdefemptysue.pyx cython-0.20.1+1~202203241016-9537/tests/broken/cdefemptysue.pyx --- cython-0.20.1+1~201611251650-6686/tests/broken/cdefemptysue.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/broken/cdefemptysue.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -cdef extern from "cdefemptysue.h": - - cdef struct spam: - pass - - ctypedef union eggs: - pass - - cdef enum ham: - pass - -cdef extern spam s -cdef extern eggs e -cdef extern ham h diff -Nru cython-0.20.1+1~201611251650-6686/tests/broken/cdefexternblock.pyx cython-0.20.1+1~202203241016-9537/tests/broken/cdefexternblock.pyx --- cython-0.20.1+1~201611251650-6686/tests/broken/cdefexternblock.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/broken/cdefexternblock.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ -cdef extern from "cheese.h": - - ctypedef int camembert - - struct roquefort: - int x - - char *swiss - - void cheddar() - - class external.runny [object runny_obj]: - cdef int a - def __init__(self): - pass - -cdef runny r -r = x -r.a = 42 diff -Nru cython-0.20.1+1~201611251650-6686/tests/buffers/bufaccess.pyx cython-0.20.1+1~202203241016-9537/tests/buffers/bufaccess.pyx --- cython-0.20.1+1~201611251650-6686/tests/buffers/bufaccess.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/buffers/bufaccess.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -13,8 +13,6 @@ from cpython.ref cimport Py_INCREF, Py_DECREF cimport cython -__test__ = {} - import sys #import re exclude = []#re.compile('object').search] @@ -27,8 +25,7 @@ def testcase(func): for e in exclude: if e(func.__name__): - return func - __test__[func.__name__] = func.__doc__ + func.__doc__ = "" # disable the test return func @@ -599,7 +596,7 @@ acquired R 25 released R - >>> [str(x) for x in R.recieved_flags] # Works in both py2 and py3 + >>> [str(x) for x in R.received_flags] # Works in both py2 and py3 ['FORMAT', 'INDIRECT', 'ND', 'STRIDES'] """ cdef object[unsigned short int, ndim=3] buf = obj @@ -612,7 +609,7 @@ >>> writable(R) acquired R released R - >>> [str(x) for x in R.recieved_flags] # Py2/3 + >>> [str(x) for x in R.received_flags] # Py2/3 ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] """ cdef object[unsigned short int, ndim=3] buf = obj @@ -626,7 +623,7 @@ acquired A released A 2 - >>> [str(x) for x in A.recieved_flags] # Py2/3 + >>> [str(x) for x in A.received_flags] # Py2/3 ['FORMAT', 'ND', 'STRIDES'] Check that the suboffsets were patched back prior to release. @@ -641,7 +638,7 @@ >>> A = IntMockBuffer(None, range(4)) >>> c_contig(A) 2 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS'] """ return buf[2] @@ -649,12 +646,12 @@ @testcase def c_contig_2d(object[int, ndim=2, mode='c'] buf): """ - Multi-dim has seperate implementation + Multi-dim has separate implementation >>> A = IntMockBuffer(None, range(12), shape=(3,4)) >>> c_contig_2d(A) 7 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS'] """ return buf[1, 3] @@ -665,7 +662,7 @@ >>> A = IntMockBuffer(None, range(4)) >>> f_contig(A) 2 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS'] """ return buf[2] @@ -678,7 +675,7 @@ >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4)) >>> f_contig_2d(A) 7 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS'] """ return buf[3, 1] @@ -959,6 +956,8 @@ def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (x).ob_refcnt @@ -991,15 +990,16 @@ See comments on printbuf_object above. >>> a, b = [1, 2, 3], [4, 5, 6] - >>> get_refcount(a), get_refcount(b) - (2, 2) + >>> rca1, rcb1 = get_refcount(a), get_refcount(b) + >>> rca1 == rcb1 + True >>> addref(a) >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists... - >>> get_refcount(a), get_refcount(b) - (3, 2) + >>> get_refcount(a) == rca1+1, get_refcount(b) == rcb1 + (True, True) >>> assign_to_object(A, 1, b) - >>> get_refcount(a), get_refcount(b) - (2, 3) + >>> get_refcount(a) == rca1, get_refcount(b) == rcb1+1 + (True, True) >>> decref(b) """ buf[idx] = obj @@ -1010,15 +1010,14 @@ See comments on printbuf_object above. >>> a, b = [1, 2, 3], {4:23} - >>> get_refcount(a) - 2 + >>> rc1 = get_refcount(a) >>> addref(a) >>> A = ObjectMockBuffer(None, [b, a]) - >>> get_refcount(a) - 3 + >>> get_refcount(a) == rc1+1 + True >>> assign_temporary_to_object(A) - >>> get_refcount(a) - 2 + >>> get_refcount(a) == rc1 + True >>> printbuf_object(A, (2,)) {4: 23} 2 @@ -1103,7 +1102,7 @@ >>> bufdefaults1(A) acquired A released A - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES'] """ pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/buffers/buffmt.pyx cython-0.20.1+1~202203241016-9537/tests/buffers/buffmt.pyx --- cython-0.20.1+1~201611251650-6686/tests/buffers/buffmt.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/buffers/buffmt.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,11 +1,8 @@ from __future__ import unicode_literals +import struct # Tests buffer format string parsing. -__test__ = {} -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func from libc cimport stdlib @@ -37,7 +34,7 @@ cdef class MockBuffer: cdef Py_ssize_t zero cdef Py_ssize_t minusone - cdef object format + cdef bytes format cdef object itemsize def __init__(self, format, itemsize): @@ -55,7 +52,6 @@ info.format = self.format info.itemsize = self.itemsize -@testcase def _int(fmt): """ >>> _int("i") @@ -77,14 +73,12 @@ """ cdef object[int] buf = MockBuffer(fmt, sizeof(int)) -@testcase def _ulong(fmt): """ >>> _ulong("L") """ cdef object[unsigned long] buf = MockBuffer(fmt, sizeof(unsigned long)) -@testcase def wrongsize(): """ >>> wrongsize() @@ -95,7 +89,6 @@ """ cdef object[float] buf = MockBuffer("f", 1) -@testcase def _obj(fmt): """ >>> _obj("O") @@ -117,6 +110,9 @@ int c int d +ctypedef struct LongString: + char[90198] c + cdef struct CharIntCFloat: char a int b @@ -147,7 +143,6 @@ char d int e, f, g -@testcase def char3int(fmt): """ >>> char3int("ciii") @@ -162,7 +157,7 @@ >>> char3int("c3xiii") >>> char3int("cxxxiii") - Standard alignment (assming int size is 4) + Standard alignment (assuming int size is 4) >>> char3int("=c3xiii") >>> char3int("=ciii") Traceback (most recent call last): @@ -180,7 +175,15 @@ cdef object obj = MockBuffer(fmt, sizeof(Char3Int)) cdef object[Char3Int, ndim=1] buf = obj -@testcase + +def long_string(fmt): + """ + >>> long_string("90198s") + """ + cdef object obj = MockBuffer(fmt, sizeof(LongString)) + cdef object[LongString, ndim=1] buf = obj + + def unpacked_struct(fmt): """ Native formats: @@ -204,7 +207,6 @@ cdef struct ComplexTest: ComplexFloat a, b, c -@testcase def complex_test(fmt): """ >>> complex_test("ZfZfZf") @@ -222,7 +224,6 @@ cdef object[ComplexTest] buf1 = obj -@testcase def alignment_string(fmt, exc=None): """ >>> alignment_string("@i") @@ -244,7 +245,6 @@ print "fail" -@testcase def int_and_long_are_same(): """ >>> int_and_long_are_same() @@ -259,7 +259,6 @@ double real float imag -@testcase def mixed_complex_struct(): """ Triggering a specific execution path for this case. @@ -297,7 +296,6 @@ char b int c -@testcase def packed_struct(fmt): """ Assuming int is four bytes: @@ -320,7 +318,6 @@ """ cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct)) -@testcase def partially_packed_struct(fmt): """ Assuming int is four bytes: @@ -348,7 +345,6 @@ cdef object[PartiallyPackedStruct] buf = MockBuffer( fmt, sizeof(PartiallyPackedStruct)) -@testcase def partially_packed_struct_2(fmt): """ Assuming int is four bytes: @@ -366,7 +362,7 @@ Traceback (most recent call last): ... ValueError: Buffer dtype mismatch; next field is at offset 8 but 5 expected - + >>> partially_packed_struct_2("ccici") Traceback (most recent call last): ... @@ -384,7 +380,6 @@ char[3] d -@testcase def packed_struct_with_strings(fmt): """ >>> packed_struct_with_strings("T{f:a:i:b:5s:c:3s:d:}") @@ -393,6 +388,59 @@ fmt, sizeof(PackedStructWithCharArrays)) +ctypedef struct PackedStructWithArrays: + double a[16] + double b[16] + double c + +ctypedef struct UnpackedStructWithArrays: + int a + float b[8] + float c + unsigned long long d + int e[5] + int f + int g + double h[4] + int i + +ctypedef struct PackedStructWithNDArrays: + double a + double b[2][2] + float c + float d + + +def packed_struct_with_arrays(fmt): + """ + >>> packed_struct_with_arrays("T{(16)d:a:(16)d:b:d:c:}") + """ + + cdef object[PackedStructWithArrays] buf = MockBuffer( + fmt, sizeof(PackedStructWithArrays)) + + +def unpacked_struct_with_arrays(fmt): + """ + >>> if struct.calcsize('P') == 8: # 64 bit + ... unpacked_struct_with_arrays("T{i:a:(8)f:b:f:c:Q:d:(5)i:e:i:f:i:g:xxxx(4)d:h:i:i:}") + ... elif struct.calcsize('P') == 4: # 32 bit + ... unpacked_struct_with_arrays("T{i:a:(8)f:b:f:c:Q:d:(5)i:e:i:f:i:g:(4)d:h:i:i:}") + """ + + cdef object[UnpackedStructWithArrays] buf = MockBuffer( + fmt, sizeof(UnpackedStructWithArrays)) + + +def packed_struct_with_ndarrays(fmt): + """ + >>> packed_struct_with_ndarrays("T{d:a:(2,2)d:b:f:c:f:d:}") + """ + + cdef object[PackedStructWithNDArrays] buf = MockBuffer( + fmt, sizeof(PackedStructWithNDArrays)) + + # TODO: empty struct # TODO: Incomplete structs # TODO: mixed structs diff -Nru cython-0.20.1+1~201611251650-6686/tests/buffers/mockbuffers.pxi cython-0.20.1+1~202203241016-9537/tests/buffers/mockbuffers.pxi --- cython-0.20.1+1~201611251650-6686/tests/buffers/mockbuffers.pxi 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/buffers/mockbuffers.pxi 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,4 @@ from libc cimport stdlib -from libc cimport stdio cimport cpython.buffer import sys @@ -18,35 +17,36 @@ cdef object format, offset cdef void* buffer cdef Py_ssize_t len, itemsize - cdef int ndim cdef Py_ssize_t* strides cdef Py_ssize_t* shape cdef Py_ssize_t* suboffsets cdef object label, log + cdef int ndim + cdef bint writable - cdef readonly object recieved_flags, release_ok + cdef readonly object received_flags, release_ok cdef public object fail - def __init__(self, label, data, shape=None, strides=None, format=None, offset=0): + def __init__(self, label, data, shape=None, strides=None, format=None, writable=True, offset=0): # It is important not to store references to data after the constructor # as refcounting is checked on object buffers. + cdef Py_ssize_t x, s, cumprod, itemsize self.label = label self.release_ok = True - self.log = "" + self.log = u"" self.offset = offset - self.itemsize = self.get_itemsize() + self.itemsize = itemsize = self.get_itemsize() + self.writable = writable if format is None: format = self.get_default_format() if shape is None: shape = (len(data),) if strides is None: strides = [] cumprod = 1 - rshape = list(shape) - rshape.reverse() - for s in rshape: + for s in shape[::-1]: strides.append(cumprod) cumprod *= s strides.reverse() - strides = [x * self.itemsize for x in strides] + strides = [x * itemsize for x in strides] suboffsets = [-1] * len(shape) datashape = [len(data)] p = data @@ -59,8 +59,10 @@ self.ndim = len(datashape) shape = datashape self.buffer = self.create_indirect_buffer(data, shape) - suboffsets = [0] * (self.ndim-1) + [-1] - strides = [sizeof(void*)] * (self.ndim-1) + [self.itemsize] + suboffsets = [0] * self.ndim + suboffsets[-1] = -1 + strides = [sizeof(void*)] * self.ndim + strides[-1] = itemsize self.suboffsets = self.list_to_sizebuf(suboffsets) else: # strided and/or simple access @@ -73,7 +75,7 @@ except AttributeError: pass self.format = format - self.len = len(data) * self.itemsize + self.len = len(data) * itemsize self.strides = self.list_to_sizebuf(strides) self.shape = self.list_to_sizebuf(shape) @@ -117,6 +119,7 @@ return buf cdef Py_ssize_t* list_to_sizebuf(self, l): + cdef Py_ssize_t i, x cdef size_t n = len(l) * sizeof(Py_ssize_t) cdef Py_ssize_t* buf = stdlib.malloc(n) for i, x in enumerate(l): @@ -127,16 +130,19 @@ if self.fail: raise ValueError("Failing on purpose") - self.recieved_flags = [] + self.received_flags = [] cdef int value for name, value in available_flags: if (value & flags) == value: - self.recieved_flags.append(name) + self.received_flags.append(name) + + if flags & cpython.buffer.PyBUF_WRITABLE and not self.writable: + raise BufferError(f"Writable buffer requested from read-only mock: {' | '.join(self.received_flags)}") buffer.buf = (self.buffer + (self.offset * self.itemsize)) buffer.obj = self buffer.len = self.len - buffer.readonly = 0 + buffer.readonly = not self.writable buffer.format = self.format buffer.ndim = self.ndim buffer.shape = self.shape @@ -145,29 +151,29 @@ buffer.itemsize = self.itemsize buffer.internal = NULL if self.label: - msg = "acquired %s" % self.label - print msg - self.log += msg + "\n" + msg = f"acquired {self.label}" + print(msg) + self.log += msg + u"\n" def __releasebuffer__(MockBuffer self, Py_buffer* buffer): if buffer.suboffsets != self.suboffsets: self.release_ok = False if self.label: - msg = "released %s" % self.label - print msg - self.log += msg + "\n" + msg = f"released {self.label}" + print(msg) + self.log += msg + u"\n" def printlog(self): - print self.log[:-1] + print(self.log[:-1]) def resetlog(self): - self.log = "" + self.log = u"" cdef int write(self, char* buf, object value) except -1: raise Exception() cdef get_itemsize(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed: {self.__class__}") cdef get_default_format(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed {self.__class__}") cdef class CharMockBuffer(MockBuffer): cdef int write(self, char* buf, object value) except -1: @@ -239,10 +245,10 @@ self.label = label def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags): - raise Exception("acquiring %s" % self.label) + raise Exception(f"acquiring {self.label}") def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer): - raise Exception("releasing %s" % self.label) + raise Exception(f"releasing {self.label}") # # Structs @@ -329,13 +335,12 @@ def print_offsets(*args, size, newline=True): - sys.stdout.write(' '.join([str(item // size) for item in args])) - if newline: - sys.stdout.write('\n') + sys.stdout.write(' '.join([str(item // size) for item in args]) + ('\n' if newline else '')) def print_int_offsets(*args, newline=True): print_offsets(*args, size=sizeof(int), newline=newline) + shape_5_3_4_list = [[list(range(k * 12 + j * 4, k * 12 + j * 4 + 4)) for j in range(3)] for k in range(5)] diff -Nru cython-0.20.1+1~201611251650-6686/tests/buffers/userbuffer.pyx cython-0.20.1+1~202203241016-9537/tests/buffers/userbuffer.pyx --- cython-0.20.1+1~201611251650-6686/tests/buffers/userbuffer.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/buffers/userbuffer.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,75 @@ + +__doc__ = u""" +>>> b1 = UserBuffer1() +>>> m1 = memoryview(b1) +>>> m1.tolist() +[0, 1, 2, 3, 4] +>>> del m1, b1 + +>>> b2 = UserBuffer2() +>>> m2 = memoryview(b2) +UserBuffer2: getbuffer +>>> m2.tolist() +[5, 6, 7, 8, 9] +>>> del m2, b2 +UserBuffer2: release +""" + +cdef extern from *: + ctypedef struct Py_buffer # redeclared + enum: PyBUF_SIMPLE + int PyBuffer_FillInfo(Py_buffer *, object, void *, Py_ssize_t, bint, int) except -1 + int PyObject_GetBuffer(object, Py_buffer *, int) except -1 + void PyBuffer_Release(Py_buffer *) + +cdef char global_buf[5] +global_buf[0:5] = [0, 1, 2, 3, 4] + +cdef class UserBuffer1: + + def __getbuffer__(self, Py_buffer* view, int flags): + PyBuffer_FillInfo(view, None, global_buf, 5, 1, flags) + +cdef class UserBuffer2: + cdef char buf[5] + + def __cinit__(self): + self.buf[0:5] = [5, 6, 7, 8, 9] + + def __getbuffer__(self, Py_buffer* view, int flags): + print('UserBuffer2: getbuffer') + PyBuffer_FillInfo(view, self, self.buf, 5, 0, flags) + + def __releasebuffer__(self, Py_buffer* view): + print('UserBuffer2: release') + + +cdef extern from *: + ctypedef struct PyBuffer"Py_buffer": + void *buf + Py_ssize_t len + bint readonly + +cdef class _memoryview: + + """ + Memory + """ + + cdef PyBuffer view + + def __cinit__(self, obj): + cdef Py_buffer *view = &self.view + PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) + + def __dealloc__(self): + cdef Py_buffer *view = &self.view + PyBuffer_Release(view ) + + def __getbuffer__(self, Py_buffer *view, int flags): + PyBuffer_FillInfo(view, self, + self.view.buf, self.view.len, + self.view.readonly, flags) + def tolist(self): + cdef char *b = self.view.buf + return [b[i] for i in range(self.view.len)] diff -Nru cython-0.20.1+1~201611251650-6686/tests/bugs.txt cython-0.20.1+1~202203241016-9537/tests/bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/bugs.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,7 @@ # This file contains tests corresponding to unresolved bugs, # which will be skipped in the normal testing run. +setuptools_reimport class_attribute_init_values_T18 unsignedbehaviour_T184 missing_baseclass_in_predecl_T262 @@ -12,6 +13,9 @@ inherited_final_method cimport_alias_subclass +# PEP-489 is currently disabled +run.mod__spec__ + # CPython regression tests that don't current work: pyregr.test_signal pyregr.test_capi diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_c_in_temp.srctree cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_c_in_temp.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_c_in_temp.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_c_in_temp.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,30 @@ + +PYTHON setup.py build_ext --inplace --cython-c-in-temp +PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"' +PYTHON check_paths.py + +############# setup.py ############# + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'mymodule', + sources=['mymodule.pyx'], + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## mymodule.pyx ######## + +test_string = "TEST" + +######## check_paths.py ######## + +import os +assert not os.path.exists("mymodule.c") diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_cplus.srctree cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_cplus.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_cplus.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_cplus.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,34 @@ +# tag: cpp + +PYTHON setup.py build_ext --inplace --cython-cplus +PYTHON -c "import a; a.use_vector([1,2,3])" + +######## setup.py ######## + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'a', + sources=['a.pyx'], + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## a.pyx ######## + +from libcpp.vector cimport vector + +def use_vector(L): + try: + v = new vector[int]() + for a in L: + v.push_back(a) + return v.size() + finally: + del v diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_include_dirs.srctree cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_include_dirs.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/build_ext_cython_include_dirs.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/build_ext_cython_include_dirs.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,50 @@ + +PYTHON setup.py build_ext --inplace --cython-include-dirs=./headers1 --include-dirs=./headers2 +PYTHON -c 'import mymodule; assert mymodule.test_string == "TEST"; assert mymodule.header_value1 == 1; assert mymodule.header_value2 == 2; assert mymodule.header_value3 == 3; assert mymodule.header_value4 == 4' + +############# setup.py ############# + +from Cython.Distutils.extension import Extension +from Cython.Build import build_ext +from distutils.core import setup + +setup( + name='Hello world app', + ext_modules = [ + Extension( + name = 'mymodule', + sources=['mymodule.pyx'], + cython_include_dirs=['headers3'], + include_dirs=['./headers4'] + ) + ], + cmdclass={'build_ext': build_ext}, +) + +######## mymodule.pyx ######## + +include "myheader1.pxi" +include "myheader2.pxi" +include "myheader3.pxi" +include "myheader4.pxi" +header_value1 = test_value1 +header_value2 = test_value2 +header_value3 = test_value3 +header_value4 = test_value4 +test_string = "TEST" + +######## headers1/myheader1.pxi ######## + +cdef int test_value1 = 1 + +######## headers2/myheader2.pxi ######## + +cdef int test_value2 = 2 + +######## headers3/myheader3.pxi ######## + +cdef int test_value3 = 3 + +######## headers4/myheader4.pxi ######## + +cdef int test_value4 = 4 diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/common_include_dir.srctree cython-0.20.1+1~202203241016-9537/tests/build/common_include_dir.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/common_include_dir.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/common_include_dir.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -16,13 +16,19 @@ import sys from Cython.Build.Dependencies import cythonize +import os from distutils.core import setup +# os x on CI specifically seems to crash with nthreads>0 +osx_on_ci = (sys.platform == "darwin" and os.getenv("CI")) + # Test concurrent safety if multiprocessing is available. -# (In particular, TravisCI does not support spawning processes from tests.) +# (In particular, CI providers like Travis and Github Actions do not support spawning processes from tests.) nthreads = 0 -if not hasattr(sys, 'pypy_version_info'): +if not (hasattr(sys, 'pypy_version_info') or + (hasattr(sys, 'implementation') and sys.implementation.name == 'graalpython') or + osx_on_ci): try: import multiprocessing multiprocessing.Pool(2).close() @@ -72,9 +78,10 @@ assert opt == '-c' count = 0 regex = re.compile(pattern) - for line in open(file): - if regex.search(line): - count += 1 + with open(file) as fid: + for line in fid: + if regex.search(line): + count += 1 print(count) sys.exit(count == 0) else: diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_cython.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_cython.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_cython.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_cython.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,58 @@ +PYTHON -c "import cythonize_tests" +PYTHON -c "import cython_tests" + +######## cythonize_tests.py ######## + +from Cython.Build.Cythonize import main as cythonize + +for test_case in ["cython.pyx", "src2/cython.pyx", "src/cython/helper.pyx"]: + try: + cythonize([test_case]) + except ValueError: + pass + else: + assert False, "ValueError not raised - forbidding cythonize "+test_case+" doesn't work" + +for test_case in ["notcython.pyx", "my_module/cython.pyx", "cythontest/helper.pyx"]: + try: + cythonize([test_case]) + except ValueError: + assert False, "ValueError raised - cythonize "+test_case+" should work" + else: + pass + +######## cython_tests.py ######## + + +from Cython.Compiler.Main import main as cython +import sys + +for test_case in ["cython.pyx", "scr2/cython.pyx", "src/cython/helper.pyx"]: + sys.argv=["cython", test_case] #cython.py will extract parameters from sys.argv + try: + cython(command_line=1) + except ValueError: + pass + else: + assert False, "ValueError not raised - forbidding cython "+test_case+" doesn't work" + + +for test_case in ["notcython.pyx", "my_module/cython.pyx", "cythontest/helper.pyx"]: + sys.argv=["cython", test_case] #cython.py will extract parameters from sys.argv + try: + cython([test_case]) + except ValueError: + assert False, "ValueError raised - cython "+test_case+" should work" + else: + pass + + +######## cython.pyx ######## +######## my_module/__init__.py ######## +######## my_module/cython.pyx ######## +######## notcython.pyx ######## +######## src2/cython.pyx ######## +######## src/cython/__init__.py ######## +######## src/cython/helper.pyx ######## +######## cythontest/__init__.py ######## +######## cythontest/helper.pyx ######## diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_newer_files.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_newer_files.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_newer_files.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_newer_files.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,96 @@ +""" +PYTHON test.py +""" + +######## a.pyx ######## + +######## test.py ######## + +import os.path +import time + +from Cython.Utils import GENERATED_BY_MARKER_BYTES, clear_function_caches, clear_method_caches +from Cython.Build.Dependencies import cythonize, DependencyTree +import Cython.Build.Dependencies + +getmtime = os.path.getmtime + +def wait_for_newer_mtime(filename, old_mtime): + mtime = old_mtime + while mtime <= old_mtime: + os.utime(filename, None) + mtime = getmtime(filename) + return mtime + + +# test the mtime waiting itself +with open("test_file.txt", 'wb') as f: + pass +orig_mtime = getmtime("test_file.txt") +wait_for_newer_mtime("test_file.txt", orig_mtime) +assert orig_mtime < getmtime("test_file.txt") + + +def fresh_cythonize(*args): + clear_function_caches() + clear_method_caches(DependencyTree.timestamp) + Cython.Build.Dependencies._dep_tree = None + cythonize(*args) + + +assert not os.path.exists("a.c") + +# new +fresh_cythonize("*.pyx") +assert os.path.isfile("a.c") +mtime = getmtime("a.c") + +# already exists +fresh_cythonize("*.pyx") +assert mtime == getmtime("a.c") + +# outdated +wait_for_newer_mtime("a.pyx", mtime) +assert mtime < getmtime("a.pyx") +fresh_cythonize("*.pyx") +new_mtime = getmtime("a.c") +assert mtime < new_mtime + +# now up to date +fresh_cythonize("*.pyx") +assert new_mtime == getmtime("a.c") + +# different Cython version (even though newer) +marker = b"/* Generated by Cython " +assert GENERATED_BY_MARKER_BYTES.startswith(marker) # safety belt +with open("a.c", "rb") as f: + content = f.read() + +assert content.startswith(GENERATED_BY_MARKER_BYTES) +content = marker + b"123" + content[len(marker):] + +with open("a.c", "wb") as f: + f.write(content) +wait_for_newer_mtime("a.c", new_mtime) + +other_cython_mtime = getmtime("a.c") +assert mtime < new_mtime < other_cython_mtime + +fresh_cythonize("*.pyx") +latest_mtime = getmtime("a.c") +assert mtime < new_mtime < other_cython_mtime <= latest_mtime + +with open("a.c", "rb") as f: + assert f.read(len(GENERATED_BY_MARKER_BYTES)) == GENERATED_BY_MARKER_BYTES # file was rewritten + +# now up to date +fresh_cythonize("*.pyx") +assert mtime < new_mtime < other_cython_mtime <= latest_mtime == getmtime("a.c") + +# force regeneration with environment variable +os.environ["CYTHON_FORCE_REGEN"] = "1" +time.sleep(0.1) + +assert latest_mtime == getmtime("a.c") +fresh_cythonize("*.pyx") +assert latest_mtime < getmtime("a.c") diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_options.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_options.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_options.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_options.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -1,3 +1,5 @@ +# mode: run + PYTHON setup.py build_ext --inplace PYTHON -c "import a" @@ -5,11 +7,35 @@ from Cython.Build.Dependencies import cythonize +import sys from distutils.core import setup +try: + # io.StringIO cannot handle 'str' in Py2 + from StringIO import StringIO +except ImportError: + from io import StringIO + +old_stderr = sys.stderr +captured = sys.stderr = StringIO() +try: + setup( + ext_modules = cythonize( + "*.pyx", include_path=['subdir'], + compiler_directives={'cdivision': True}, + show_all_warnings=True, + ), + ) + output = sys.stderr.getvalue() +finally: + sys.stderr = old_stderr + sys.stderr.write(captured.getvalue()) + +assert "Unraisable exception in function" in output, output + -setup( - ext_modules = cythonize("*.pyx", include_path=['subdir'], compiler_directives={'cdivision': True}), -) +######## subdir/x.pxd ######## + +######## subdir/y.pxi ######## ######## a.pyx ######## @@ -22,8 +48,6 @@ assert mod_int_c(-1, 10) < 0 - -######## subdir/x.pxd ######## - -######## subdir/y.pxi ######## - +# unraisable exceptions should produce a warning +cdef int no_exc_propagate(): + raise TypeError() diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_pep420_namespace.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_pep420_namespace.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_pep420_namespace.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_pep420_namespace.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,59 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup, Extension + +setup( + ext_modules=cythonize([ + Extension("nsp.m1.a", ["nsp/m1/a.pyx"]), + Extension("nsp.m2.b", ["nsp/m2/b.pyx"]), + Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"]), + ]), +) + +######## nsp/m1/__init__.py ######## + +######## nsp/m1/a.pyx ######## + +cdef class A: + pass + +######## nsp/m1/a.pxd ######## + +cdef class A: + pass + +######## nsp/m2/__init__.py ######## + +######## nsp/m2/b.pyx ######## + +from nsp.m1.a cimport A +from nsp.m3.c.d cimport D + +cdef class B(A): + pass + +######## nsp/m3/__init__.py ######## + +######## nsp/m3/c/d.pyx ######## + +cdef class D: + pass + +######## nsp/m3/c/d.pxd ######## + +cdef class D: + pass + +######## runner.py ######## + +from nsp.m1.a import A +from nsp.m2.b import B +from nsp.m3.c.d import D + +a = A() +b = B() diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_rename_ext.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_rename_ext.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_rename_ext.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_rename_ext.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,38 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "from pkg import b; assert b.test() == 43" + +######## setup.py ######## + +from Cython.Build import cythonize + +from distutils.core import setup, Extension + +extensions = [ + Extension('pkg.b', sources=['pkg/a.pyx', 'pkg/alib.c'], + include_dirs=['pkg']) +] + +setup( + ext_modules = cythonize(extensions) +) + +######## pkg/__init__.py ######## + +######## pkg/a.pyx ######## + +cdef extern from "alib.h": + int c_function(int x) + +def test(): + return c_function(42) + + +######## pkg/alib.c ######## + +int c_function(int x) { + return x + 1; +} + +######## pkg/alib.h ######## + +int c_function(int x); diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_script_package.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_script_package.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_script_package.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_script_package.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -11,7 +11,7 @@ # __init__.py compilation isn't supported in Py 3.[012] import pkg.sub.test assert pkg.sub.test.TEST == 'pkg.sub.test' - assert '.py' not in pkg.sub.test.__file__ + assert not pkg.sub.test.__file__.rstrip('oc').endswith('.py') ######## test.py ######## diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_script.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_script.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_script.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_script.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -1,8 +1,8 @@ ''' PYTHON -m Cython.Build.Cythonize -i '**/*_test.py' -PYTHON -c "import cy_test; assert cy_test.TEST == 'cy_test', cy_test.TEST; assert '.py' not in cy_test.__file__, cy_test.__file__" -PYTHON -c "import pkg.cy_test; assert pkg.cy_test.TEST == 'pkg.cy_test', pkg.cy_test.TEST; assert '.py' not in pkg.cy_test.__file__, pkg.cy_test.__file__" -PYTHON -c "import pkg.sub.cy_test; assert pkg.sub.cy_test.TEST == 'pkg.sub.cy_test', pkg.sub.cy_test.TEST; assert '.py' not in pkg.sub.cy_test.__file__, pkg.cy_test.__file__" +PYTHON -c "import cy_test; assert cy_test.TEST == 'cy_test', cy_test.TEST; assert not cy_test.__file__.rstrip('oc').endswith('.py'), cy_test.__file__" +PYTHON -c "import pkg.cy_test; assert pkg.cy_test.TEST == 'pkg.cy_test', pkg.cy_test.TEST; assert not pkg.cy_test.__file__.rstrip('oc').endswith('.py'), pkg.cy_test.__file__" +PYTHON -c "import pkg.sub.cy_test; assert pkg.sub.cy_test.TEST == 'pkg.sub.cy_test', pkg.sub.cy_test.TEST; assert not pkg.sub.cy_test.__file__.rstrip('oc').endswith('.py'), pkg.cy_test.__file__" ''' ######## cy_test.py ######## diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,45 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import not_annotated; not_annotated.check()" +PYTHON -c "import default_annotated; default_annotated.check()" +PYTHON -c "import fullc_annotated; fullc_annotated.check()" +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup + +setup( + ext_modules = cythonize(["not_annotated.pyx"], language_level=3) + + cythonize(["default_annotated.pyx"], annotate=True, language_level=3) + + cythonize(["fullc_annotated.pyx"], annotate='fullc', language_level=3) +) +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate_via_cli.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate_via_cli.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate_via_cli.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate_via_cli.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,36 @@ +CYTHONIZE -i -3 not_annotated.pyx +PYTHON -c "import not_annotated; not_annotated.check()" +CYTHONIZE -i -3 --annotate default_annotated.pyx +PYTHON -c "import default_annotated; default_annotated.check()" +CYTHONIZE -i -3 --annotate-fullc fullc_annotated.pyx +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate_via_Options.srctree cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate_via_Options.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/cythonize_with_annotate_via_Options.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/cythonize_with_annotate_via_Options.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,27 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from Cython.Compiler import Options + +Options.annotate = 'fullc' + +from distutils.core import setup + +setup( + ext_modules = cythonize(["fullc_annotated.pyx"], language_level=3) +) + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: + +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/depfile_numpy.srctree cython-0.20.1+1~202203241016-9537/tests/build/depfile_numpy.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/depfile_numpy.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/depfile_numpy.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,58 @@ +# tag: numpy + +""" +CYTHONIZE -M dep_np.pyx +PYTHON check_np.py +""" + +######## dep_np.pyx ######## + +cimport numpy as np +np.import_array() + + + +######## check_np.py ######## + +import os.path +import re + +import numpy as np +import Cython + +with open("dep_np.c.dep", "r") as f: + contents = f.read().replace('\\\n', ' ').replace('\n', ' ') + +contents = contents.split() + +cy_prefix, _ = os.path.split(Cython.__file__) +contents = [fname.replace(cy_prefix, "cy_prefix") for fname in contents] + +np_prefix, _ = os.path.split(np.__file__) +contents = [fname.replace(np_prefix, "np_prefix") for fname in contents] + +# filter out the version number from `np_prefix/__init__.cython-30.pxd`. +contents = [re.sub('[.]cython-[0-9]+', '', entry) for entry in contents] + +contents = [path.split(os.sep) for path in contents] +contents.sort() + +expected = [path.split('/') for path in [ + 'cy_prefix/Includes/cpython/object.pxd', + 'cy_prefix/Includes/cpython/ref.pxd', + 'cy_prefix/Includes/cpython/type.pxd', + 'cy_prefix/Includes/libc/stdio.pxd', + 'cy_prefix/Includes/libc/string.pxd', + 'dep_np.c:', + 'dep_np.pyx', +]] + +# Also account for legacy numpy versions, which do not ship +# `__init__.pxd` hence the fallback is used: +if ['cy_prefix', 'Includes', 'numpy', '__init__.pxd'] in contents: + expected.append(['cy_prefix', 'Includes', 'numpy', '__init__.pxd']) +else: + expected.append(['np_prefix', '__init__.pxd']) + +expected.sort() +assert contents == expected, contents diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/depfile_package.srctree cython-0.20.1+1~202203241016-9537/tests/build/depfile_package.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/depfile_package.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/depfile_package.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,57 @@ +""" +CYTHONIZE -i pkg --depfile +PYTHON package_test.py +""" + +######## package_test.py ######## + +import os.path + +with open(os.path.join("pkg", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +assert sorted(contents.split()) == sorted(['test.c:', os.path.join('sub', 'incl.pxi'), 'test.pxd', 'test.pyx']), contents + + +with open(os.path.join("pkg", "sub", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +contents = [os.path.relpath(entry, '.') + if os.path.isabs(entry) else entry for entry in contents.split()] +assert sorted(contents) == sorted(['test.c:', 'incl.pxi', 'test.pyx', os.path.join('pkg', 'test.pxd')]), contents # last is really one level up + + +######## pkg/__init__.py ######## + + +######## pkg/test.pyx ######## + +TEST = "pkg.test" + +include "sub/incl.pxi" + +cdef object get_str(): + return TEST + + +######## pkg/test.pxd ######## + +cdef object get_str() + + +######## pkg/sub/__init__.py ######## + + +######## pkg/sub/test.pyx ######## +# cython: language_level=3 + +from ..test cimport get_str + +include 'incl.pxi' + +TEST = 'pkg.sub.test' + + +######## pkg/sub/incl.pxi ######## + +pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/depfile.srctree cython-0.20.1+1~202203241016-9537/tests/build/depfile.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/depfile.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/depfile.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,33 @@ +""" +CYTHONIZE -M foo.pyx +PYTHON check.py +""" + +######## foo.pyx ######## + +from bar cimport empty + +include "baz.pxi" + +empty() +print(foo()) + + +######## baz.pxi ######## + +def foo(): + return "foo" + + +######## bar.pxd ######## + +cdef inline void empty(): + print("empty") + + +######## check.py ######## + +with open("foo.c.dep", "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +assert sorted(contents.split()) == ['bar.pxd', 'baz.pxi', 'foo.c:', 'foo.pyx'], contents diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/dotted.filename.modules.pyx cython-0.20.1+1~202203241016-9537/tests/build/dotted.filename.modules.pyx --- cython-0.20.1+1~201611251650-6686/tests/build/dotted.filename.modules.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/dotted.filename.modules.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,7 @@ +# mode: compile +# tag: warnings + +_WARNINGS=""" +1:0: Dotted filenames ('dotted.filename.modules.pxd') are deprecated. Please use the normal Python package directory layout. +1:0: Dotted filenames ('dotted.filename.modules.pyx') are deprecated. Please use the normal Python package directory layout. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/inline_distutils.srctree cython-0.20.1+1~202203241016-9537/tests/build/inline_distutils.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/inline_distutils.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/inline_distutils.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -38,4 +38,4 @@ from my_lib cimport x -print x +print(x) diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/module_api.srctree cython-0.20.1+1~202203241016-9537/tests/build/module_api.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/module_api.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/module_api.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -108,6 +108,19 @@ if (!sys_modules) return; mod = PyInit_a(); if (!mod) return; +#if PY_VERSION_HEX >= 0x03050000 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) + /* FIXME: this is incomplete and users shouldn't have to do this in the first place... */ + if (!PyModule_Check(mod)) { + /* In PEP 489 multi-phase init, PyInit_a returns PyModuleDef */ + PyModuleDef *mdef = (PyModuleDef*)mod; + PyObject *modname = PyUnicode_FromString("a"); + if (!modname) return; + mod = PyModule_NewObject(modname); + Py_DECREF(modname); + if (!mod) return; + PyModule_ExecDef(mod, mdef); + } +#endif PyDict_SetItemString(sys_modules, (char*)"a", mod); } #endif diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/package_compilation.srctree cython-0.20.1+1~202203241016-9537/tests/build/package_compilation.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/package_compilation.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/package_compilation.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ PYTHON setup.py build_ext --inplace -PYTHON -c "import toppkg; assert '.py' not in toppkg.__file__; assert toppkg.PACKAGE == 1" -PYTHON -c "import toppkg.subpkg; assert '.py' not in toppkg.__file__; assert '.py' not in toppkg.subpkg.__file__; assert toppkg.subpkg.PACKAGE == 2" +PYTHON -c "import toppkg; assert not toppkg.__file__.rstrip('oc').endswith('.py'); assert toppkg.PACKAGE == 1" +PYTHON -c "import toppkg.subpkg; assert not toppkg.__file__.rstrip('oc').endswith('.py'); assert not toppkg.subpkg.__file__.rstrip('oc').endswith('.py'); assert toppkg.subpkg.PACKAGE == 2" PYTHON -c "import toppkg.a; assert toppkg.a.MODULE == 'a'" PYTHON -c "from toppkg.subpkg import a; assert a.MODULE == 'subpkg.a'" diff -Nru cython-0.20.1+1~201611251650-6686/tests/build/setuptools_reimport.srctree cython-0.20.1+1~202203241016-9537/tests/build/setuptools_reimport.srctree --- cython-0.20.1+1~201611251650-6686/tests/build/setuptools_reimport.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/build/setuptools_reimport.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,23 @@ +# mode: run +# tag: setuptools + +PYTHON setup.py + + +######## module.pyx ######## + +cimport cython + + +######## import_cython.py ######## + +import Cython.Compiler.Main + + +######## setup.py ######## + +from setuptools.sandbox import run_setup +run_setup('import_cython.py', ['egg_info']) + +from Cython.Build import cythonize +cythonize('module.pyx') diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/branch_hints.pyx cython-0.20.1+1~202203241016-9537/tests/compile/branch_hints.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/branch_hints.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/branch_hints.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,91 @@ +# mode: compile +# tag: if, unlikely + +cimport cython + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_simple(x): + if x: + x = 2 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_return(x): + if x: + return 1 + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else(x): + if x: + raise TypeError() + else: + return 1 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'likely']", +) +def if_else_raise(x): + if x: + return 1 + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else_raise(x): + if x: + raise ValueError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +def if_elif_raise_else_raise(x): + if x: + raise ValueError() + elif not x: + raise AttributeError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", + "//IfClauseNode[@branch_hint = 'unlikely']//GILStatNode", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +cpdef int nogil_if_raise(int x) nogil except -1: + if x: + raise TypeError() + elif not x: + raise ValueError() + else: + x = 2 diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/buildenv.pyx cython-0.20.1+1~202203241016-9537/tests/compile/buildenv.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/buildenv.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/buildenv.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,143 @@ +""" +Non-test that prints debug information about the current build environment. +""" + +from __future__ import print_function + +import os +import sys +from distutils import sysconfig + + +cdef extern from *: + """ + #ifndef PyLong_SHIFT + #define PyLong_SHIFT 0 + typedef int digit; + typedef int sdigit; + #endif + #ifndef PyLong_BASE + #define PyLong_BASE 0 + #endif + #ifndef PyLong_MASK + #define PyLong_MASK 0 + #endif + #ifndef SIZEOF_UINTPTR_T + #define SIZEOF_UINTPTR_T 0 + #endif + #ifndef SIZEOF_OFF_T + #define SIZEOF_OFF_T 0 + #endif + """ + # Python runtime + cdef long PY_VERSION_HEX + + # Cython config + cdef int CYTHON_COMPILING_IN_CPYTHON + cdef int CYTHON_COMPILING_IN_LIMITED_API + cdef int CYTHON_COMPILING_IN_PYPY + cdef int CYTHON_USE_PYLONG_INTERNALS + cdef int CYTHON_USE_PYLIST_INTERNALS + cdef int CYTHON_USE_UNICODE_INTERNALS + cdef int CYTHON_USE_UNICODE_WRITER + cdef int CYTHON_AVOID_BORROWED_REFS + cdef int CYTHON_ASSUME_SAFE_MACROS + cdef int CYTHON_USE_TYPE_SLOTS + cdef int CYTHON_UNPACK_METHODS + cdef int CYTHON_FAST_THREAD_STATE + cdef int CYTHON_FAST_PYCALL + cdef int CYTHON_PEP489_MULTI_PHASE_INIT + cdef int CYTHON_USE_TP_FINALIZE + + # C and platform specifics + cdef int SIZEOF_INT + cdef int SIZEOF_LONG + cdef int SIZEOF_SIZE_T + cdef int SIZEOF_LONG_LONG + cdef int SIZEOF_VOID_P + cdef int SIZEOF_OFF_T + cdef int SIZEOF_UINTPTR_T + + # PyLong internals + cdef long PyLong_BASE + cdef long PyLong_MASK + cdef int PyLong_SHIFT + cdef int digit + cdef int sdigit + + +def config_var(name, default=''): + return sysconfig.get_config_var(name) or default + +get_env = os.environ.get + + +print(f"""Python build environment: +Python {sys.version_info} +PY_VERSION_HEX 0x{PY_VERSION_HEX:X} + +CYTHON_COMPILING_IN_CPYTHON {CYTHON_COMPILING_IN_CPYTHON} +CYTHON_COMPILING_IN_LIMITED_API {CYTHON_COMPILING_IN_LIMITED_API} +CYTHON_COMPILING_IN_PYPY {CYTHON_COMPILING_IN_PYPY} + +CYTHON_USE_PYLONG_INTERNALS {CYTHON_USE_PYLONG_INTERNALS} +CYTHON_USE_PYLIST_INTERNALS {CYTHON_USE_PYLIST_INTERNALS} +CYTHON_USE_UNICODE_INTERNALS {CYTHON_USE_UNICODE_INTERNALS} +CYTHON_USE_UNICODE_WRITER {CYTHON_USE_UNICODE_WRITER} +CYTHON_AVOID_BORROWED_REFS {CYTHON_AVOID_BORROWED_REFS} +CYTHON_ASSUME_SAFE_MACROS {CYTHON_ASSUME_SAFE_MACROS} +CYTHON_USE_TYPE_SLOTS {CYTHON_USE_TYPE_SLOTS} +CYTHON_UNPACK_METHODS {CYTHON_UNPACK_METHODS} +CYTHON_FAST_THREAD_STATE {CYTHON_FAST_THREAD_STATE} +CYTHON_FAST_PYCALL {CYTHON_FAST_PYCALL} +CYTHON_PEP489_MULTI_PHASE_INIT {CYTHON_PEP489_MULTI_PHASE_INIT} +CYTHON_USE_TP_FINALIZE {CYTHON_USE_TP_FINALIZE} + +PyLong_BASE 0x{PyLong_BASE:X} +PyLong_MASK 0x{PyLong_MASK:X} +PyLong_SHIFT {PyLong_SHIFT} +sizeof(digit) {sizeof(digit)} +sizeof(sdigit) {sizeof(sdigit)} +sys.int_info {getattr(sys, 'int_info', '-')} +sys.getsizeof(1, 2**14, 2**15, 2**29, 2**30, 2**59, 2**60, 2**64) {tuple(sys.getsizeof(n) for n in (1, 2**14, 2**15, 2**29, 2**30, 2**59, 2**60, 2**64))} + +SIZEOF_INT {SIZEOF_INT} ({sizeof(int)}) +SIZEOF_LONG {SIZEOF_LONG} ({sizeof(long)}) +SIZEOF_SIZE_T {SIZEOF_SIZE_T} ({sizeof(Py_ssize_t)}, {getattr(sys, 'maxsize', getattr(sys, 'maxint', None))}) +SIZEOF_LONG_LONG {SIZEOF_LONG_LONG} ({sizeof(long long)}) +SIZEOF_VOID_P {SIZEOF_VOID_P} ({sizeof(void*)}) +SIZEOF_UINTPTR_T {SIZEOF_UINTPTR_T} ({sizeof(unsigned int *)}) +SIZEOF_OFF_T {SIZEOF_OFF_T} + +Paths: +sys.executable = {sys.executable} +sys.exec_prefix = {sys.exec_prefix} +sys.base_exec_prefix = {getattr(sys, 'base_exec_prefix', "")} +sys.prefix = {sys.prefix} +sys.path = {sys.path} +PYTHONPATH (env) = {get_env('PYTHONPATH', '')} +PYTHONHOME (env) = {get_env('PYTHONHOME', '')} + +Distutils: +INCDIR = {sysconfig.get_python_inc()} +LIBS = {config_var('LIBS')} +LIBDIR = {config_var('LIBDIR')} +LIBPL = {config_var('LIBPL')} +Python LIBRARY = {config_var('LIBRARY')} +LINKFORSHARED = {config_var('LINKFORSHARED')} + +CC (distutils) = {config_var('CC')} +CC (env) = {get_env('CC', '')} +CFLAGS (distutils) = {config_var('CFLAGS')} +CFLAGS (env) = {get_env('CFLAGS', '')} +LINKCC (distutils) = {config_var('LINKCC')} +LINKCC (env) = {get_env('LINKCC', '')} + +Encodings: +sys maxunicode = {sys.maxunicode} +LANG (env) = {get_env('LANG', '')} +PYTHONIOENCODING (env) = {get_env('PYTHONIOENCODING', '')} +sys stdout encoding = {sys.stdout.encoding} +sys default encoding = {sys.getdefaultencoding()} +sys FS encoding = {sys.getfilesystemencoding()} +""") diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/builtinbuffer.py cython-0.20.1+1~202203241016-9537/tests/compile/builtinbuffer.py --- cython-0.20.1+1~201611251650-6686/tests/compile/builtinbuffer.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/builtinbuffer.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,6 @@ +# mode: compile +import cython + +@cython.cclass +class BuiltinRef: + cython.declare(pybuf = 'Py_buffer') diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/callingconvention.srctree cython-0.20.1+1~202203241016-9537/tests/compile/callingconvention.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/callingconvention.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/callingconvention.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -17,7 +17,7 @@ ######## callingconvention.pyx ######## # mode: compile -cdef extern from "external_callingconvention.h": +cdef extern from "callingconvention.h": pass cdef extern int f1() @@ -35,10 +35,19 @@ p3 = f3 p4 = f4 + +######## callingconvention.h ######## + +#define DLL_EXPORT +#include "external_callingconvention.h" + + ######## external_callingconvention.h ######## #ifndef DL_IMPORT #define DL_IMPORT(t) t +#elif defined(DLL_EXPORT) + #define DL_IMPORT(t) DL_EXPORT(t) #endif #ifdef __cplusplus diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cascmp.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cascmp.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cascmp.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cascmp.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,17 +0,0 @@ -# mode: compile - -cdef void foo(): - cdef int bool, int1=0, int2=0, int3=0, int4=0 - cdef object obj1, obj2, obj3, obj4 - obj1 = 1 - obj2 = 2 - obj3 = 3 - obj4 = 4 - bool = int1 < int2 < int3 - bool = obj1 < obj2 < obj3 - bool = int1 < int2 < obj3 - bool = obj1 < 2 < 3 - bool = obj1 < 2 < 3 < 4 - bool = int1 < (int2 == int3) < int4 - -foo() diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cast_ctypedef_array_T518.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cast_ctypedef_array_T518.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cast_ctypedef_array_T518.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cast_ctypedef_array_T518.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 518 +# ticket: t518 # mode: compile cdef extern from "cast_ctypedef_array_T518_helper.h": diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cdefemptysue.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cdefemptysue.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cdefemptysue.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cdefemptysue.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,43 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from *: + """ + struct spam { int a; }; + struct flat_spam { int a; }; + typedef struct { int a; } flat_spam_type; + + typedef union { int a; long b; } eggs; + typedef union { int a; long b; } flat_eggs; + + enum ham { TOAST }; + enum flat_ham { FLAT_TOAST }; + """ + + cdef struct spam: + pass + + cdef struct flat_spam: pass + + ctypedef struct flat_spam_type: pass + + ctypedef union eggs: + pass + + ctypedef union flat_eggs: pass + + cdef enum ham: + pass + + cdef enum flat_ham: pass + + +cdef extern spam s +cdef extern flat_spam fs +cdef extern flat_spam_type fst + +cdef extern eggs e +cdef extern flat_eggs fe + +cdef extern ham h +cdef extern flat_ham fh diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cdefexternblock.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cdefexternblock.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cdefexternblock.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cdefexternblock.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,23 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from "cheese.h": + + ctypedef int camembert + + struct roquefort: + int x + + char *swiss + + void cheddar() + + # FIXME: find a real declaration here. + #class external.runny [object runny_obj]: + # cdef int a + # def __init__(self): + # pass + + +#cdef runny r = runny() +#r.a = 42 diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cdef_syntax.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cdef_syntax.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cdef_syntax.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cdef_syntax.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -9,3 +9,6 @@ cdef with_semi(): cdef int i; + +def use_cdef(): + &no_semi, &with_semi diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cimported_class_base.srctree cython-0.20.1+1~202203241016-9537/tests/compile/cimported_class_base.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/cimported_class_base.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cimported_class_base.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,43 @@ +PYTHON setup.py build_ext --inplace + +######## setup.py ######## + +from Cython.Build import cythonize +from Cython.Distutils.extension import Extension + +import sys +sys.path.append("path") + +ext_modules = [ + Extension("importer", ["importer.pyx"]), +] + +ext_modules = cythonize(ext_modules, include_path=["include"]) + + +######## pkg/__init__.py ######## + + +######## pkg/a.pxd ######## + +cdef class A(object): + pass + +######## importer.pyx ######## + +cimport pkg.a +cimport pkg.a as a +cimport pkg.a as a_by_another_name +from pkg cimport a as from_cimported_a + +cdef class A1(a.A): + pass + +cdef class A2(a_by_another_name.A): + pass + +cdef class A3(pkg.a.A): + pass + +cdef class A4(from_cimported_a.A): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cimportfrom_T248.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cimportfrom_T248.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cimportfrom_T248.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cimportfrom_T248.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 248 +# ticket: t248 # mode: compile from ewing8 cimport (Foo, diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cimport_package_module_T4.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cimport_package_module_T4.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cimport_package_module_T4.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cimport_package_module_T4.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 4 +# ticket: t4 # mode: compile from a cimport b diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cnamespec.h cython-0.20.1+1~202203241016-9537/tests/compile/cnamespec.h --- cython-0.20.1+1~201611251650-6686/tests/compile/cnamespec.h 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cnamespec.h 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -int c_a, c_b; diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cnamespec.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cnamespec.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cnamespec.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cnamespec.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,9 @@ # mode: compile -cdef extern from "cnamespec.h": +cdef extern from *: + """ + int c_a, c_b; + """ int a "c_a", b "c_b" cdef struct foo "c_foo": @@ -14,7 +17,8 @@ cdef double d "c_d" cdef foo *p global b - d = spam(a, f) + if i: + d = spam(a, f) cdef foo q q.i = 7 p = &q diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/complex_annotations.pyx cython-0.20.1+1~202203241016-9537/tests/compile/complex_annotations.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/complex_annotations.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/complex_annotations.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,7 @@ +# mode: compile + +# Complex numbers defined in annotations weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +cdef f(x: complex): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/complex_decorators.pyx cython-0.20.1+1~202203241016-9537/tests/compile/complex_decorators.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/complex_decorators.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/complex_decorators.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,10 @@ +# mode: compile + +cimport cython + +# Complex numbers defined in "cython.locals" weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +@cython.locals(x=complex) +cdef f(x): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/const_decl.pyx cython-0.20.1+1~202203241016-9537/tests/compile/const_decl.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/const_decl.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/const_decl.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,12 +1,17 @@ # mode: compile -cdef const_args(const int a, const int *b, const (int*) c, int *const d): +cdef const_args(const int a, const int *b, const (int*) c, int *const d, int **const e, int *const *f): print a print b[0] - b = NULL # OK, the pointer itself is not const - c[0] = 4 # OK, the value is not const - d[0] = 7 # OK, the value is not const + b = NULL # OK, the pointer itself is not const + c[0] = 4 # OK, the value is not const + d[0] = 7 # OK, the value is not const + e[0][0] = 1 # OK, the value is not const + e[0] = NULL # OK, the pointed pointer is not const + f[0][0] = 1 # OK, the value is not const + f = NULL # OK, the pointer is not const def call_const_args(x): cdef int k = x - const_args(x, &k, &k, &k) + cdef int* arr = [x] + const_args(x, &k, &k, &k, &arr, &arr) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_class_redefinition.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_class_redefinition.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_class_redefinition.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_class_redefinition.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,5 @@ -# tag: cpp # mode: compile +# tag: cpp, warnings cdef extern from "templates.h": cdef cppclass TemplateTest1[T]: diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cppenum.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cppenum.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cppenum.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cppenum.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,31 @@ +# mode: compile +# tag: cpp,cpp11 + + +cpdef enum class Spam: + a, b + c + d + e + f = 42 + + +cpdef enum class Cheese(unsigned int): + x = 1 + y = 2 + + +cdef enum struct parrot_state: + alive = 1 + dead = 0 + + +cdef void eggs(): + cdef Spam s1 + s1 = Spam.a + s2 = Spam.b + + cdef Cheese c1 + c1 = Cheese.x + +eggs() diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_enums.h cython-0.20.1+1~202203241016-9537/tests/compile/cpp_enums.h --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_enums.h 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_enums.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -enum Enum1 { - Item1, - Item2 -}; - -namespace Namespace1 { - enum Enum2 { - Item3, - Item4 - }; -} diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_enums.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_enums.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_enums.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_enums.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,27 +0,0 @@ -# tag: cpp -# mode: compile - -cdef extern from "cpp_enums.h": - cdef enum Enum1: - Item1 - Item2 - -a = Item1 -b = Item2 - -cdef Enum1 x, y -x = Item1 -y = Item2 - -cdef extern from "cpp_enums.h" namespace "Namespace1": - cdef enum Enum2: - Item3 - Item4 - -c = Item3 -d = Item4 - -cdef Enum2 z, w -z = Item3 -w = Item4 - diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_nogil.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_nogil.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_nogil.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_nogil.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -16,3 +16,8 @@ with nogil: NoGilTest1().doSomething() NoGilTest2().doSomething() + +# We can override nogil methods as with gil methods. +cdef cppclass WithGilSubclass(NoGilTest1): + void doSomething() with gil: + print "have the gil" diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_rvalue_reference_binding.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_rvalue_reference_binding.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_rvalue_reference_binding.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_rvalue_reference_binding.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +# tag: cpp, cpp11 +# mode: compile + +cdef extern from *: + """ + template + void accept(T&& x) { (void) x; } + """ + cdef void accept[T](T&& x) + +cdef int make_int_py() except *: + # might raise Python exception (thus needs a temp) + return 1 + +cdef int make_int_cpp() except +: + # might raise C++ exception (thus needs a temp) + return 1 + +def test_func_arg(): + # won't compile if move() isn't called on the temp: + accept(make_int_py()) + accept(make_int_cpp()) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_temp_assignment.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_temp_assignment.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_temp_assignment.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_temp_assignment.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,98 @@ +# tag: cpp,cpp11 +# mode: compile +# tag: no-cpp-locals +# TODO cpp_locals works fine with the standard library that comes with gcc11 +# but not with gcc8. Therefore disable the test for now + +cdef extern from *: + """ + class NoAssignIterator { + public: + explicit NoAssignIterator(int pos) : pos_(pos) {} + NoAssignIterator(NoAssignIterator&) = delete; + NoAssignIterator(NoAssignIterator&&) {} + NoAssignIterator& operator=(NoAssignIterator&) = delete; + NoAssignIterator& operator=(NoAssignIterator&&) { return *this; } + // Default constructor of temp variable is needed by Cython + // as of 3.0a6. + NoAssignIterator() : pos_(0) {} + int operator*() { + return pos_; + } + NoAssignIterator operator++() { + return NoAssignIterator(pos_ + 1); + } + int operator!=(NoAssignIterator other) { + return pos_ != other.pos_; + } + int pos_; + }; + class NoAssign { + public: + NoAssign() {} + NoAssign(NoAssign&) = delete; + NoAssign(NoAssign&&) {} + NoAssign& operator=(NoAssign&) = delete; + NoAssign& operator=(NoAssign&&) { return *this; } + void func() {} + NoAssignIterator begin() { + return NoAssignIterator(0); + } + NoAssignIterator end() { + return NoAssignIterator(2); + } + }; + + NoAssign get_NoAssign_Py() { + return NoAssign(); + } + NoAssign get_NoAssign_Cpp() { + return NoAssign(); + } + + """ + cdef cppclass NoAssignIterator: + int operator*() + NoAssignIterator operator++() + int operator!=(NoAssignIterator) + + cdef cppclass NoAssign: + void func() + NoAssignIterator begin() + NoAssignIterator end() + + # might raise Python exception (thus needs a temp) + NoAssign get_NoAssign_Py() except * + # might raise C++ exception (thus needs a temp) + NoAssign get_NoAssign_Cpp() except + + +cdef internal_cpp_func(NoAssign arg): + pass + +def test_call_to_function(): + # will fail to compile if move constructors aren't used + internal_cpp_func(get_NoAssign_Py()) + internal_cpp_func(get_NoAssign_Cpp()) + +def test_assignment_to_name(): + # will fail if move constructors aren't used + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + +def test_assignment_to_scope(): + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + def inner(): + value.func() + +cdef class AssignToClassAttr: + cdef NoAssign attr + def __init__(self): + self.attr = get_NoAssign_Py() + self.attr = get_NoAssign_Cpp() + +def test_generator_cpp_iterator_as_temp(): + for i in get_NoAssign_Py(): + yield i diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_templates_nested.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_templates_nested.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_templates_nested.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_templates_nested.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,18 @@ +# tag: cpp +# mode: compile + +from libcpp.vector cimport vector + +cdef extern from *: + cdef cppclass Foo[T]: + pass + + cdef cppclass Bar: + pass + +cdef vector[vector[int]] a +cdef vector[vector[const int]] b +cdef vector[vector[vector[int]]] c +cdef vector[vector[vector[const int]]] d +cdef Foo[Foo[Bar]] e +cdef Foo[Foo[const Bar]] f diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cpp_templates.pyx cython-0.20.1+1~202203241016-9537/tests/compile/cpp_templates.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/cpp_templates.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cpp_templates.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ # tag: cpp # mode: compile -# ticket: 767 +# ticket: t767 cdef extern from "templates.h": cdef cppclass TemplateTest1[T]: diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/create_extension.srctree cython-0.20.1+1~202203241016-9537/tests/compile/create_extension.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/create_extension.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/create_extension.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,24 @@ +PYTHON setup.py build_ext --inplace + +######## setup.py ######## + +from Cython.Build import cythonize +from Cython.Distutils.extension import Extension + +ext_modules = [ + Extension("foo", ["foo.pyx"]), +] + +# Example documented in docs/src/reference/compilation.rst +from Cython.Build.Dependencies import default_create_extension + +def my_create_extension(template, kwds): + libs = kwds.get('libraries', []) + ["mylib"] + kwds['libraries'] = libs + return default_create_extension(template, kwds) + +ext_modules = cythonize(ext_modules, create_extension=my_create_extension) + +assert ext_modules[0].libraries == ["mylib"] + +######## foo.pyx ######## diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/crunchytype.h cython-0.20.1+1~202203241016-9537/tests/compile/crunchytype.h --- cython-0.20.1+1~201611251650-6686/tests/compile/crunchytype.h 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/crunchytype.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ - -struct CrunchyType { - int number; - PyObject* string; -}; diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/crunchytype.pxd cython-0.20.1+1~202203241016-9537/tests/compile/crunchytype.pxd --- cython-0.20.1+1~201611251650-6686/tests/compile/crunchytype.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/crunchytype.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,10 @@ -cdef extern from "crunchytype.h": +cdef extern from *: + """ + struct CrunchyType { + int number; + PyObject* string; + }; + """ cdef class crunchytype.Crunchy [ object CrunchyType ]: cdef int number cdef object string diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_cimport.pxd cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_cimport.pxd --- cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_cimport.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_cimport.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,15 @@ +# Verify defined before function prototype +cdef (int, int) get_a_ctuple() + +# Verify defined before typedef +ctypedef (int, double) int_double + +# Verify typedef defined +cdef int_double tuple_global = (1, 2.) + +# Verify defined before opt args +cdef void test_opt_args((double, int) x=*) + +# Verify defined before class declaration +cdef class CTupleClass: + cdef void get_a_ctuple(self, (double, double) x) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_cimport_T1427.pyx cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_cimport_T1427.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_cimport_T1427.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_cimport_T1427.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# ticket: 1427 +# mode: compile + +cimport ctuple_cimport + +# Verify same tuple defined in multiple pxd not redeclared +ctypedef (int, double) int_double + +ctuple_cimport.get_a_ctuple() diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_unused_T3543.pyx cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_unused_T3543.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/ctuple_unused_T3543.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/ctuple_unused_T3543.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# ticket: 3543 +# mode: compile + +# Views define unused ctuples, including (long,) +from cython cimport view + +# Implicitly generate a ctuple (long,) +obj = None +obj or (1,) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/ctypedef_public_class_T355.pyx cython-0.20.1+1~202203241016-9537/tests/compile/ctypedef_public_class_T355.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/ctypedef_public_class_T355.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/ctypedef_public_class_T355.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 355 +# ticket: t355 # mode: compile ctypedef public class Time [type MyTime_Type, object MyTimeObject]: diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cython_compiled_folding.pxd cython-0.20.1+1~202203241016-9537/tests/compile/cython_compiled_folding.pxd --- cython-0.20.1+1~201611251650-6686/tests/compile/cython_compiled_folding.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cython_compiled_folding.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1 @@ +from libc.math cimport sin, cos, sqrt, tan, log diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/cython_compiled_folding.py cython-0.20.1+1~202203241016-9537/tests/compile/cython_compiled_folding.py --- cython-0.20.1+1~201611251650-6686/tests/compile/cython_compiled_folding.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/cython_compiled_folding.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,39 @@ +# mode: compile + +# libc sin, cos and sqrt cimported in the pxd file + +import cython +from cython import compiled + +if not cython.compiled: + from math import sin + +if cython.compiled: + pass +else: + from math import cos + +if "aa" == "bb": + pass +elif cython.compiled: + pass +elif True: + from math import sqrt + +if "aa" == "bb": + pass +elif compiled: + pass +else: + from math import tan + +# log10 isn't defined in the pxd file +from math import log10 + +@cython.test_fail_if_path_exists("//FromImportStatNode//ImportNode") +@cython.test_assert_path_exists("//AddNode") +def import_log(x, y): + if compiled: + return x+y + else: + from math import log diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/declarations.srctree cython-0.20.1+1~202203241016-9537/tests/compile/declarations.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/declarations.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/declarations.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -17,7 +17,7 @@ ######## declarations.pyx ######## # mode: compile -cdef extern from "external_declarations.h": +cdef extern from "declarations.h": pass cdef extern char *cp @@ -48,10 +48,19 @@ f() g() + +######## declarations.h ######## + +#define DLL_EXPORT +#include "external_declarations.h" + + ######## external_declarations.h ######## #ifndef DL_IMPORT #define DL_IMPORT(t) t +#elif defined(DLL_EXPORT) + #define DL_IMPORT(t) DL_EXPORT(t) #endif #ifdef __cplusplus diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/ellipsis_T488.pyx cython-0.20.1+1~202203241016-9537/tests/compile/ellipsis_T488.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/ellipsis_T488.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/ellipsis_T488.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 488 +# ticket: t488 # mode: compile #from ... import foo diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/extern_packed_struct.pyx cython-0.20.1+1~202203241016-9537/tests/compile/extern_packed_struct.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/extern_packed_struct.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/extern_packed_struct.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -# mode: compile - -cdef extern from *: - cdef packed struct MyStruct: - char a diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/finally_GH1744.pyx cython-0.20.1+1~202203241016-9537/tests/compile/finally_GH1744.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/finally_GH1744.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/finally_GH1744.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,113 @@ +# mode: compile + +# This caused a "maximum recursion depth exceeded" at some point, +# see https://github.com/cython/cython/issues/1744 + +cdef inline bint g(int x, int y): return True + +cdef cython_bug(int u): + try: + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + g(u, u) + finally: + g(u, u) + +cython_bug(1) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/find_pxd.srctree cython-0.20.1+1~202203241016-9537/tests/compile/find_pxd.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/find_pxd.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/find_pxd.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -6,12 +6,13 @@ from Cython.Distutils.extension import Extension import sys -sys.path.append("path") +sys.path.insert(0, "path") ext_modules = [ Extension("a", ["a.pyx"]), Extension("b", ["b.pyx"]), Extension("c", ["c.pyx"]), + Extension("d", ["d.pyx"]), ] ext_modules = cythonize(ext_modules, include_path=["include"]) @@ -37,3 +38,15 @@ ######## path/c.pxd ######## +++syntax error just to show that this file is not actually cimported+++ + +######## path/numpy/__init__.pxd ######## + +# gh-2905: This should be found before Cython/Includes/numpy/__init__.pxd + +ctypedef int my_type + +######## d.pyx ######## + +cimport numpy + +cdef numpy.my_type foo diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/funcptr.pyx cython-0.20.1+1~202203241016-9537/tests/compile/funcptr.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/funcptr.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/funcptr.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -# mode: compile - -cdef int grail(): - cdef int (*spam)() - spam = &grail - spam = grail - spam() - -ctypedef int funcptr_t() - -cdef inline funcptr_t* dummy(): - return &grail diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/fused_redeclare_T3111.pyx cython-0.20.1+1~202203241016-9537/tests/compile/fused_redeclare_T3111.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/fused_redeclare_T3111.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/fused_redeclare_T3111.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,38 @@ +# ticket: 3111 +# mode: compile +# tag: warnings + +ctypedef unsigned char npy_uint8 +ctypedef unsigned short npy_uint16 + + +ctypedef fused dtype_t: + npy_uint8 + +ctypedef fused dtype_t_out: + npy_uint8 + npy_uint16 + + +def foo(dtype_t[:] a, dtype_t_out[:, :] b): + pass + + +# The primary thing we're trying to test here is the _absence_ of the warning +# "__pyxutil:16:4: '___pyx_npy_uint8' redeclared". The remaining warnings are +# unrelated to this test. +_WARNINGS = """ +# cpdef redeclaration bug, from TestCythonScope.pyx +25:10: 'cpdef_method' redeclared +36:10: 'cpdef_cname_method' redeclared + +# from MemoryView.pyx +323:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310 +323:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310 +979:29: Ambiguous exception value, same as default return value: 0 +979:29: Ambiguous exception value, same as default return value: 0 +1004:46: Ambiguous exception value, same as default return value: 0 +1004:46: Ambiguous exception value, same as default return value: 0 +1094:29: Ambiguous exception value, same as default return value: 0 +1094:29: Ambiguous exception value, same as default return value: 0 +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/fused_unused.pyx cython-0.20.1+1~202203241016-9537/tests/compile/fused_unused.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/fused_unused.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/fused_unused.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# mode: compile +# tag: fused + +# This previously lead to a crash due to an empty module body. + +ctypedef fused cinteger: + int + long + Py_ssize_t diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/fused_wraparound.pyx cython-0.20.1+1~202203241016-9537/tests/compile/fused_wraparound.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/fused_wraparound.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/fused_wraparound.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +# mode: compile +# tag: fused, werror + +""" +Very short test for https://github.com/cython/cython/issues/3492 +(Needs its own file since werror fails on the main fused-test files) +wraparound and boundscheck directives shouldn't break the fused +dispatcher function +""" + +cimport cython + +ctypedef fused fused_t: + str + int + long + complex + +@cython.wraparound(False) +@cython.boundscheck(False) +def func(fused_t a, cython.floating b): + return a, b diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/future_imports.pyx cython-0.20.1+1~202203241016-9537/tests/compile/future_imports.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/future_imports.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/future_imports.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -7,3 +7,6 @@ pass from __future__ import nested_scopes ; from __future__ import nested_scopes + +from __future__ import print_function +print(end='') diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/libc_stdio.pyx cython-0.20.1+1~202203241016-9537/tests/compile/libc_stdio.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/libc_stdio.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/libc_stdio.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,16 +1,17 @@ # mode: compile -cimport libc +cimport libc.stdio from libc cimport stdio from libc.stdio cimport printf, puts, fputs, putchar, fputc, putc, stdout -libc.stdio.printf("hello %s\n", b"world") -stdio.printf("hello %s\n", b"world") -printf("hello %s\n", b"world") -printf("printf_output %d %d\n", 1, 2) -puts("puts_output") -fputs("fputs_output", stdout) -putchar(b'z') -fputc(b'x', stdout) -putc(b'c', stdout) +with nogil: + libc.stdio.printf("hello %s\n", b"world") + stdio.printf("hello %s\n", b"world") + printf("hello %s\n", b"world") + printf("printf_output %d %d\n", 1, 2) + puts("puts_output") + fputs("fputs_output", stdout) + putchar(b'z') + fputc(b'x', stdout) + putc(b'c', stdout) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/min_async.pyx cython-0.20.1+1~202203241016-9537/tests/compile/min_async.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/min_async.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/min_async.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +# mode: compile +# tag: pep492, await + +# Need to include all utility code ! + +async def sleep(x): + pass + + +async def call(): + await sleep(1) + yield diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/packed_structs.pyx cython-0.20.1+1~202203241016-9537/tests/compile/packed_structs.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/packed_structs.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/packed_structs.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,10 @@ +# mode: compile + +cdef extern from *: + cdef packed struct MyStruct: + char a + +cdef public packed struct PublicStruct: + int a + unsigned char b + int c diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/posix_pxds.pyx cython-0.20.1+1~202203241016-9537/tests/compile/posix_pxds.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/posix_pxds.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/posix_pxds.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,19 +1,33 @@ # tag: posix # mode: compile +# This file is generated by `Tools/gen_tests_for_posix_pxds.py`. + cimport posix -cimport posix.unistd -from posix cimport unistd -from posix.unistd cimport * +cimport posix.dlfcn +from posix cimport dlfcn +from posix.dlfcn cimport * cimport posix.fcntl from posix cimport fcntl -from posix.fcntl cimport * +from posix.fcntl cimport * -cimport posix.types -from posix cimport types -from posix.types cimport * +cimport posix.ioctl +from posix cimport ioctl +from posix.ioctl cimport * + +cimport posix.mman +from posix cimport mman +from posix.mman cimport * + +cimport posix.resource +from posix cimport resource +from posix.resource cimport * + +cimport posix.select +from posix cimport select +from posix.select cimport * cimport posix.signal from posix cimport signal @@ -31,22 +45,26 @@ from posix cimport stdlib from posix.stdlib cimport * +cimport posix.strings +from posix cimport strings +from posix.strings cimport * + cimport posix.time from posix cimport time from posix.time cimport * -cimport posix.resource -from posix cimport resource -from posix.resource cimport * +cimport posix.types +from posix cimport types +from posix.types cimport * + +cimport posix.uio +from posix cimport uio +from posix.uio cimport * + +cimport posix.unistd +from posix cimport unistd +from posix.unistd cimport * cimport posix.wait from posix cimport wait from posix.wait cimport * - -cimport posix.mman -from posix cimport mman -from posix.mman cimport * - -cimport posix.dlfcn -from posix cimport dlfcn -from posix.dlfcn cimport * diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/pylong.pyx cython-0.20.1+1~202203241016-9537/tests/compile/pylong.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/pylong.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/pylong.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,7 +8,12 @@ Py_ssize_t ob_refcnt PyTypeObject *ob_type -cdef extern from "longintrepr.h": +cdef extern from "Python.h": + """ + #if PY_MAJOR_VERSION < 3 + #include "longintrepr.h" + #endif + """ cdef struct _longobject: int ob_refcnt PyTypeObject *ob_type diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/stop_async_iteration_exception_pep492.pyx cython-0.20.1+1~202203241016-9537/tests/compile/stop_async_iteration_exception_pep492.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/stop_async_iteration_exception_pep492.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/stop_async_iteration_exception_pep492.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,27 @@ +# mode: compile +# tag: pep492 + +# make sure async iterators also compile correctly without using 'await' + +cdef class AsyncIter: + cdef long i + cdef long aiter_calls + cdef long max_iter_calls + + def __init__(self, long max_iter_calls=1): + self.i = 0 + self.aiter_calls = 0 + self.max_iter_calls = max_iter_calls + + def __aiter__(self): + self.aiter_calls += 1 + return self + + async def __anext__(self): + self.i += 1 + assert self.aiter_calls <= self.max_iter_calls + + if self.i > 10: + raise StopAsyncIteration + + return self.i, self.i diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/tree_assertions.pyx cython-0.20.1+1~202203241016-9537/tests/compile/tree_assertions.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/tree_assertions.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/tree_assertions.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,20 @@ +# mode: compile + +# This is a sort of meta test - to test the functionality of "test_assert_path_exists" + +cimport cython + +@cython.test_assert_path_exists("//ReturnStatNode") +def not_in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + pass + return 1 # should pass + +@cython.test_assert_path_exists("//ReturnStatNode") +def in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + return 1 + +# it's hard to come up with a corresponding test for fail_if_path_exists.. diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/tryfinally.pyx cython-0.20.1+1~202203241016-9537/tests/compile/tryfinally.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/tryfinally.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/tryfinally.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -18,4 +18,11 @@ finally: i = 42 +def use_name_in_finally(name): + # GH3712 + try: + [] + finally: + name() + diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/types_and_names.pyx cython-0.20.1+1~202203241016-9537/tests/compile/types_and_names.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/types_and_names.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/types_and_names.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -23,3 +23,16 @@ foo(2, 3, [], [], P, P, &P) a.point("something", 3, "anything", [], "an object", P, &P) + +# Test that internally generated names do not conflict. +cdef class A_spec: + pass + +cdef class A_members: + pass + +cdef class A_methods: + pass + +cdef class A_slots: + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/verbatiminclude_cimport.srctree cython-0.20.1+1~202203241016-9537/tests/compile/verbatiminclude_cimport.srctree --- cython-0.20.1+1~201611251650-6686/tests/compile/verbatiminclude_cimport.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/verbatiminclude_cimport.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,36 @@ +PYTHON setup.py build_ext --inplace + +######## setup.py ######## + +from Cython.Build import cythonize +from distutils.core import setup + +setup( + ext_modules = cythonize("*.pyx"), +) + +######## test.pyx ######## + +from moda cimport DEFINE_A +from modb cimport DEFINE_B + +######## moda.pxd ######## + +from verbatim cimport DEFINE_ONCE as DEFINE_A + +######## modb.pxd ######## + +from verbatim cimport DEFINE_ONCE as DEFINE_B + +######## verbatim.pxd ######## + +# Check that we include this only once +cdef extern from *: + """ + #ifdef DEFINE_ONCE + #error "DEFINE_ONCE already defined" + #endif + + #define DEFINE_ONCE 1 + """ + int DEFINE_ONCE diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/volatile.pyx cython-0.20.1+1~202203241016-9537/tests/compile/volatile.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/volatile.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/volatile.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,17 @@ +# mode: compile + +cdef volatile int x = 1 + +cdef const volatile char* greeting1 = "hello world" +cdef volatile const char* greeting2 = "goodbye" + + +cdef extern from "stdlib.h": + volatile void* malloc(size_t) + +cdef volatile long* test(volatile size_t s): + cdef volatile long* arr = malloc(s) + return arr + + +test(64) diff -Nru cython-0.20.1+1~201611251650-6686/tests/compile/weakref_T276.pyx cython-0.20.1+1~202203241016-9537/tests/compile/weakref_T276.pyx --- cython-0.20.1+1~201611251650-6686/tests/compile/weakref_T276.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/compile/weakref_T276.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 276 +# ticket: t276 # mode: compile __doc__ = u""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/cygwin_bugs.txt cython-0.20.1+1~202203241016-9537/tests/cygwin_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/cygwin_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/cygwin_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,3 @@ +complex_numbers_c89_T398_long_double +complex_numbers_T305_long_double +int_float_builtins_as_casts_T400_long_double diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/break_outside_loop.pyx cython-0.20.1+1~202203241016-9537/tests/errors/break_outside_loop.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/break_outside_loop.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/break_outside_loop.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -27,12 +27,33 @@ return True +def break_after_loop(): + for _ in range(2): + pass + + if bool_result(): + break + + try: + if bool_result(): + break + except Exception: + pass + + if bool_result(): + break + + _ERRORS = u''' 4:0: break statement not inside loop 7:4: break statement not inside loop 10:4: break statement not inside loop 13:4: break statement not inside loop +15:5: break statement not inside loop 18:5: break statement not inside loop 22:4: break statement not inside loop 24:4: break statement not inside loop +35:8: break statement not inside loop +39:12: break statement not inside loop +44:8: break statement not inside loop ''' diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/bufaccess_noassignT444.pyx cython-0.20.1+1~202203241016-9537/tests/errors/bufaccess_noassignT444.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/bufaccess_noassignT444.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/bufaccess_noassignT444.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 444 +# ticket: t444 # mode: error def test(): @@ -6,5 +6,5 @@ not_assigned_to[2] = 3 _ERRORS = """ -6:20: local variable 'not_assigned_to' referenced before assignment +6:5: local variable 'not_assigned_to' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/buffertypedef_T117.pyx cython-0.20.1+1~202203241016-9537/tests/errors/buffertypedef_T117.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/buffertypedef_T117.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/buffertypedef_T117.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 117 +# ticket: t117 # mode: error ctypedef object[float] mybuffer diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/builtin_type_inheritance.pyx cython-0.20.1+1~202203241016-9537/tests/errors/builtin_type_inheritance.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/builtin_type_inheritance.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/builtin_type_inheritance.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -12,7 +12,7 @@ pass _ERRORS = """ -5:5: inheritance from PyVarObject types like 'tuple' is not currently supported -8:5: inheritance from PyVarObject types like 'bytes' is not currently supported -11:5: inheritance from PyVarObject types like 'str' is not currently supported +5:19: inheritance from PyVarObject types like 'tuple' is not currently supported +8:19: inheritance from PyVarObject types like 'bytes' is not currently supported +11:17: inheritance from PyVarObject types like 'str' is not currently supported """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/callingnonexisting_T307.pyx cython-0.20.1+1~202203241016-9537/tests/errors/callingnonexisting_T307.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/callingnonexisting_T307.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/callingnonexisting_T307.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,8 +1,8 @@ -# ticket: 307 +# ticket: t307 # mode: error nonexisting(3, with_kw_arg=4) _ERRORS = u""" -4:11: undeclared name not builtin: nonexisting +4:0: undeclared name not builtin: nonexisting """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cdef_class_properties_decorated.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cdef_class_properties_decorated.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cdef_class_properties_decorated.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cdef_class_properties_decorated.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,5 @@ # mode: error -# ticket: 264 +# ticket: t264 # tag: property, decorator @@ -34,9 +34,14 @@ def prop2(self, value): pass + @prop2.setter + def other_name(self, value): + pass + _ERRORS = """ 19:4: Property methods with additional decorators are not supported 27:4: Property methods with additional decorators are not supported 33:4: Property methods with additional decorators are not supported +37:4: Mismatching property names, expected 'prop2', got 'other_name' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cdef_func_decorators.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cdef_func_decorators.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cdef_func_decorators.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cdef_func_decorators.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,39 @@ +# mode: error +# tag: decorator + +from functools import wraps + +@wraps +cdef cant_be_decoratored(): + pass + +@wraps +cpdef also_cant_be_decorated(): + pass + +cdef class C: + @wraps + cdef still_cant_be_decorated(self): + pass + + @property + cdef property_only_works_for_extern_classes(self): + pass + + @wraps + cpdef also_still_cant_be_decorated(self): + pass + + @wraps + @wraps + cdef two_is_just_as_bad_as_one(self): + pass + +_ERRORS = """ +6:0: Cdef functions cannot take arbitrary decorators. +10:0: Cdef functions cannot take arbitrary decorators. +15:4: Cdef functions cannot take arbitrary decorators. +19:4: Cdef functions cannot take arbitrary decorators. +23:4: Cdef functions cannot take arbitrary decorators. +27:4: Cdef functions cannot take arbitrary decorators. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cdef_func_syntax.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cdef_func_syntax.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cdef_func_syntax.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cdef_func_syntax.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error + +cdef inline func() -> int: + pass + +cpdef inline func() -> int: + pass + + +_ERRORS = u""" +3:19: Return type annotation is not allowed in cdef/cpdef signatures. Please define it before the function name, as in C signatures. +6:20: Return type annotation is not allowed in cdef/cpdef signatures. Please define it before the function name, as in C signatures. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cdefkwargs.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cdefkwargs.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cdefkwargs.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cdefkwargs.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -6,10 +6,6 @@ >>> call4() """ -import sys, re -if sys.version_info >= (2,6): - __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__) - # the calls: def call2(): diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cdef_members_T517.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cdef_members_T517.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cdef_members_T517.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cdef_members_T517.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 517 +# ticket: t517 # mode: error ctypedef void* VoidP diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cfunc_directive_in_pyclass.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cfunc_directive_in_pyclass.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cfunc_directive_in_pyclass.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cfunc_directive_in_pyclass.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -7,5 +7,5 @@ pass _ERRORS = """ - 6:4: cfunc directive is not allowed here + 5:4: cfunc directive is not allowed here """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/charptr_from_temp.pyx cython-0.20.1+1~202203241016-9537/tests/errors/charptr_from_temp.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/charptr_from_temp.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/charptr_from_temp.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -53,13 +53,13 @@ _ERRORS = """ -16:8: Obtaining 'char *' from externally modifiable global Python value +16:7: Obtaining 'char *' from externally modifiable global Python value 19:9: Storing unsafe C derivative of temporary Python reference #22:8: Storing unsafe C derivative of temporary Python reference #23:5: Storing unsafe C derivative of temporary Python reference #23:15: Casting temporary Python object to non-numeric non-Python type 26:8: Storing unsafe C derivative of temporary Python reference -41:9: Obtaining 'Py_UNICODE *' from externally modifiable global Python value +41:8: Obtaining 'Py_UNICODE *' from externally modifiable global Python value 44:10: Storing unsafe C derivative of temporary Python reference 52:7: Storing unsafe C derivative of temporary Python reference 52:7: Unsafe C derivative of temporary Python reference used in conditional expression diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cimport_attributes.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cimport_attributes.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cimport_attributes.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cimport_attributes.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,32 @@ +# mode: error +# tag: cpp + + +cimport libcpp +print libcpp.no_such_attribute + +cimport libcpp.map +print libcpp.map.no_such_attribute + +from libcpp cimport vector +print vector.no_such_attribute + +from libcpp cimport vector as my_vector +print my_vector.no_such_attribute + +from libcpp cimport vector as my_vector_with_shadow +from libcpp import vector as my_vector_with_shadow +print my_vector_with_shadow.python_attribute # OK (if such a module existed at runtime) + +# Other ordering +from libcpp import map as my_map_with_shadow +from libcpp cimport map as my_map_with_shadow +print my_map_with_shadow.python_attribute # OK (if such a module existed at runtime) + + +_ERRORS = u""" +6:12: cimported module has no attribute 'no_such_attribute' +9:16: cimported module has no attribute 'no_such_attribute' +12:12: cimported module has no attribute 'no_such_attribute' +15:15: cimported module has no attribute 'no_such_attribute' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cmethbasematch.pxd cython-0.20.1+1~202203241016-9537/tests/errors/cmethbasematch.pxd --- cython-0.20.1+1~201611251650-6686/tests/errors/cmethbasematch.pxd 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cmethbasematch.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,11 @@ +cdef class Base(object): + cdef f(self) + +cdef class MissingRedeclaration(Base): + pass + +cdef class BadRedeclaration(Base): + cdef f(self) + +cdef class NarrowerReturn(Base): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cmethbasematch.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cmethbasematch.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cmethbasematch.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cmethbasematch.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,7 +8,38 @@ cdef void f(self, int x): pass +# These are declared in the pxd. +cdef class Base(object): + cdef f(self): + pass + +cdef class MissingRedeclaration(Base): + # Not declared (so assumed cdef) in the pxd. + cpdef f(self): + pass + +cdef class BadRedeclaration(Base): + # Declared as cdef in the pxd. + cpdef f(self): + pass + +cdef class UnneededRedeclaration(Base): + # This is OK, as it's not declared in the pxd. + cpdef f(self): + pass + +cdef class NarrowerReturn(Base): + # This does not require a new vtable entry. + cdef Base f(self): + pass + + _ERRORS = u""" 8: 9: Signature not compatible with previous declaration 4: 9: Previous declaration is here +# TODO(robertwb): Re-enable these errors. +#18:8: Compatible but non-identical C method 'f' not redeclared in definition part of extension type 'MissingRedeclaration' +#2:9: Previous declaration is here +#23:8: Compatible but non-identical C method 'f' not redeclared in definition part of extension type 'BadRedeclaration' +#2:9: Previous declaration is here """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/compile_time_unraisable_T370.pyx cython-0.20.1+1~202203241016-9537/tests/errors/compile_time_unraisable_T370.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/compile_time_unraisable_T370.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/compile_time_unraisable_T370.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 370 +# ticket: t370 # mode: error cdef int raiseit(): diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/const_decl_errors.pyx cython-0.20.1+1~202203241016-9537/tests/errors/const_decl_errors.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/const_decl_errors.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/const_decl_errors.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -10,22 +10,34 @@ cdef struct S: int member -cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, +cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, int **const e, int *const *f, const S *const t): a = 10 c = NULL b[0] = 100 s.member = 1000 d = NULL + e[0][0] = 1 # ok + e[0] = NULL # ok + e = NULL # nok + f[0][0] = 1 # ok + f[0] = NULL # nok + f = NULL # ok t = &s +cdef volatile object v + + _ERRORS = """ -3:5: Const base type cannot be a Python object +3:5: Const/volatile base type cannot be a Python object 8:5: Assignment to const 'x' -15:6: Assignment to const 'a' -16:6: Assignment to const 'c' +15:4: Assignment to const 'a' +16:4: Assignment to const 'c' 17:5: Assignment to const dereference 18:5: Assignment to const attribute 'member' -19:6: Assignment to const 'd' -20:6: Assignment to const 't' +19:4: Assignment to const 'd' +22:4: Assignment to const 'e' +24:5: Assignment to const dereference +26:4: Assignment to const 't' +28:5: Const/volatile base type cannot be a Python object """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/continue_outside_loop.pyx cython-0.20.1+1~202203241016-9537/tests/errors/continue_outside_loop.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/continue_outside_loop.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/continue_outside_loop.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -26,11 +26,13 @@ def bool_result(): return True -_ERRORS = u''' + +_ERRORS = ''' 4:0: continue statement not inside loop 7:4: continue statement not inside loop 10:4: continue statement not inside loop 13:4: continue statement not inside loop +15:5: continue statement not inside loop 18:5: continue statement not inside loop 22:4: continue statement not inside loop 24:4: continue statement not inside loop diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpdef_vars.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpdef_vars.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpdef_vars.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpdef_vars.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,24 @@ +# mode: error + +cpdef str a = "123" +cpdef b = 2 + +cdef class C: + cpdef float c + +def func(): + """ + >>> c = func() + >>> isinstance(c, C) or c + True + """ + cpdef d = C() + return d + + +_ERRORS = """ +3:6: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +4:6: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +7:10: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +15:10: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_bool.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_bool.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_bool.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_bool.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# tag: cpp +# mode: error + +from libcpp.string cimport string + +cdef foo(): + cdef string field + if field: # field cannot be coerced to bool + pass + +_ERRORS = u""" +8:7: Type 'string' not acceptable as a boolean +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_class_gil_GH1986.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_class_gil_GH1986.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_class_gil_GH1986.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_class_gil_GH1986.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,20 @@ +# tag: cpp +# mode: error + + +cdef cppclass Base: + __init__() nogil: + pass + +cdef cppclass Sub1(Base): + __init__(): # implicit requires GIL + pass + +cdef cppclass Sub2(Sub1): + __init__() nogil: + pass + +_ERRORS = u""" +10:4: Base constructor defined here. +14:4: Constructor cannot be called without GIL unless all base constructors can also be called without GIL +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_enum_redeclare.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_enum_redeclare.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_enum_redeclare.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_enum_redeclare.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error +# tag: cpp + +cdef enum class Spam: + a + +cdef enum class Spam: + b + +_ERRORS=""" +7:5: 'Spam' redeclared +4:5: Previous declaration is here +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cppexc_non_extern.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cppexc_non_extern.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cppexc_non_extern.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cppexc_non_extern.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +# mode: error +# tag: warnings + +cdef inline void handle_exception(): + pass + +# GH 3064 - cppfunc caused invalid code to be generated with +handle_exception +# error to prevent this +cdef test_func1(self) except +handle_exception: + pass + +# warning +cdef test_func2(self) except +: + pass + +_ERRORS = """ +9:16: Only extern functions can throw C++ exceptions. +""" + +_WARNINGS = """ +13:16: Only extern functions can throw C++ exceptions. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_no_auto_conversion.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_no_auto_conversion.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_no_auto_conversion.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_no_auto_conversion.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -19,5 +19,5 @@ _ERRORS = u""" -18:40: Cannot assign type 'long' to 'wrapped_int' +18:40: Cannot assign type 'long' to 'const wrapped_int' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_object_template.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_object_template.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_object_template.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_object_template.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ # mode: error +# tag: cpp from libcpp.vector cimport vector @@ -11,7 +12,13 @@ cdef vector[A] va va.push_back(A()) +def memview(): + import array + cdef vector[int[:]] vmv + vmv.push_back(array.array("i", [1,2,3])) + _ERRORS = u""" -9:16: Python object type 'Python object' cannot be used as a template argument -11:16: Python object type 'A' cannot be used as a template argument +10:16: Python object type 'Python object' cannot be used as a template argument +12:16: Python object type 'A' cannot be used as a template argument +17:16: Reference-counted type 'int[:]' cannot be used as a template argument """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/cpp_rvalue_reference_support.pyx cython-0.20.1+1~202203241016-9537/tests/errors/cpp_rvalue_reference_support.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/cpp_rvalue_reference_support.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/cpp_rvalue_reference_support.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,32 @@ +# mode: error +# tag: werror, cpp, cpp11 + +# These tests check for unsupported use of rvalue-references (&&) +# and should be removed or cleaned up when support is added. + +cdef int&& x + +cdef void foo(int&& x): + pass + +cdef int&& bar(): + pass + +cdef extern from *: + """ + void baz(int x, int&& y) {} + + template + void qux(const T&& x) {} + """ + cdef void baz(int x, int&& y) + cdef void qux[T](const T&& x) + + +_ERRORS=""" +7:8: C++ rvalue-references cannot be declared +9:13: Rvalue-reference as function argument not supported +12:14: Rvalue-reference as function return type not supported +22:17: Rvalue-reference as function argument not supported +23:20: Rvalue-reference as function argument not supported +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e1.pyx cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e1.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e1.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e1.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,22 @@ +# mode: error + +cimport cython + +@cython.dataclasses.dataclass(1, shouldnt_be_here=True, init=5, unsafe_hash=True) +cdef class C: + a: list = [] # mutable + b: int = cython.dataclasses.field(default=5, default_factory=int) + c: int + + def __hash__(self): + pass + +_ERRORS = """ +6:5: Arguments passed to cython.dataclasses.dataclass must be True or False +6:5: Cannot overwrite attribute __hash__ in class C +6:5: cython.dataclasses.dataclass() got an unexpected keyword argument 'shouldnt_be_here' +6:5: cython.dataclasses.dataclass takes no positional arguments +7:14: mutable default for field a is not allowed: use default_factory +8:37: cannot specify both default and default_factory +9:4: non-default argument 'c' follows default argument in dataclass __init__ +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e2.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e2.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error +# tag: dataclass + +import dataclasses + +@dataclasses.dataclass +cdef class C: + pass + +_ERRORS = """ +6:0: Cdef functions/classes cannot take arbitrary decorators. +6:0: Use '@cython.dataclasses.dataclass' on cdef classes to create a dataclass +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e3.pyx cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e3.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e3.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e3.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: compile +# tag: dataclass, warnings + +cimport cython +from dataclass import field + +@cython.dataclasses.dataclass +cdef class E: + a: int = field() + +_WARNINGS=""" +9:18: Do you mean cython.dataclasses.field instead? +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e4.pyx cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e4.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/dataclass_e4.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/dataclass_e4.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,11 @@ +# mode: error + +cimport cython + +@cython.dataclasses.dataclass +cdef class C: + a: int = cython.dataclasses.field(unexpected=True) + +_ERRORS = """ +7:49: cython.dataclasses.field() got an unexpected keyword argument 'unexpected' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/declareafteruse_T158.pyx cython-0.20.1+1~202203241016-9537/tests/errors/declareafteruse_T158.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/declareafteruse_T158.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/declareafteruse_T158.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 158 +# ticket: t158 # mode: error def mult_decl_test(): @@ -52,26 +52,19 @@ print var[0][0] cdef unsigned long long[100][100] var -# in 0.11.1 these are warnings -FUTURE_ERRORS = u""" -6:13: cdef variable 's' declared after it is used -6:16: cdef variable 'vv' declared after it is used -11:14: cdef variable 'i' declared after it is used -17:14: cdef variable 'i' declared after it is used -23:14: cdef variable 'i' declared after it is used -26:9: cdef variable 's' declared after it is used -32:17: cdef variable 't' declared after it is used -36:13: cdef variable 'r' declared after it is used -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -""" - -syntax error - _ERRORS = u""" -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -70:7: Syntax error in simple statement list +5:17: local variable 'vv' referenced before assignment +6:17: local variable 's' referenced before assignment +7:13: cdef variable 's' declared after it is used +7:16: cdef variable 'vv' declared after it is used +12:14: cdef variable 'i' declared after it is used +18:14: cdef variable 'i' declared after it is used +24:14: cdef variable 'i' declared after it is used +27:9: cdef variable 's' declared after it is used +33:17: cdef variable 't' declared after it is used +43:17: cdef variable 't' declared after it is used +50:10: cdef variable 'baz' declared after it is used +53:34: cdef variable 'var' declared after it is used """ +# FIXME not detected +#37:13: cdef variable 'r' declared after it is used diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/def_nogil.pyx cython-0.20.1+1~202203241016-9537/tests/errors/def_nogil.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/def_nogil.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/def_nogil.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -# mode: error - -def test() nogil: - pass - -_ERRORS = """ -3:0: Python function cannot be declared nogil -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/duplicate_const.pyx cython-0.20.1+1~202203241016-9537/tests/errors/duplicate_const.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/duplicate_const.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/duplicate_const.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error + +cdef extern from *: + cdef const const int a + cdef const volatile int b + cdef volatile const int c + cdef volatile volatile int d + + +_ERRORS = """ +4:9: Duplicate 'const' +7:9: Duplicate 'volatile' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e2_packedstruct_T290.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e2_packedstruct_T290.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e2_packedstruct_T290.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e2_packedstruct_T290.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 290 +# ticket: t290 # mode: error cdef packed foo: diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_addressof.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_addressof.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_addressof.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_addressof.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,29 @@ +# mode: error + +cdef class Ext: + cdef int a + cdef object o + +def f(int a): + cdef Ext e = Ext() + x = &a # ok + + cdef object o = &a # pointer != object + + po1 = &o # pointer to Python variable + po2 = &o.xyz # pointer to Python expression + po3 = &e.o # pointer to Python object + po4 = &e.a # ok (C attribute) + + po5 = &(o + 1) # pointer to non-lvalue Python expression + po6 = &(a + 1) # pointer to non-lvalue C expression + + +_ERRORS=""" +11:20: Cannot convert 'int *' to Python object +13:10: Cannot take address of Python variable 'o' +14:10: Cannot take address of Python object attribute 'xyz' +15:10: Cannot take address of Python object attribute 'o' +18:10: Taking address of non-lvalue (type Python object) +19:10: Taking address of non-lvalue (type long) +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_arrayassign.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_arrayassign.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_arrayassign.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_arrayassign.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ # mode: error +# cython: auto_pickle=False ctypedef int[1] int_array ctypedef int[2] int_array2 @@ -30,6 +31,6 @@ _ERRORS = u""" -20:2: Assignment to slice of wrong length, expected 2, got 1 -21:2: Assignment to slice of wrong length, expected 1, got 2 +21:0: Assignment to slice of wrong length, expected 2, got 1 +22:0: Assignment to slice of wrong length, expected 1, got 2 """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_assert.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_assert.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_assert.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_assert.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,25 @@ +# mode: error +# tag: assert + +def nontrivial_assert_in_nogil(int a, obj): + with nogil: + # NOK + assert obj + assert a*obj + assert obj, "abc" + + # OK + assert a + assert a*a + assert a, "abc" + assert a, u"abc" + assert a, f"123{a}xyz" + + +_ERRORS = """ +7:15: Truth-testing Python object not allowed without gil +8:15: Converting to Python object not allowed without gil +8:16: Operation not allowed without gil +8:16: Truth-testing Python object not allowed without gil +9:15: Truth-testing Python object not allowed without gil +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_ass.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_ass.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_ass.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_ass.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -11,7 +11,7 @@ _ERRORS = u""" -7:19: Cannot assign type 'char *' to 'int' -8:20: Cannot convert Python object to 'int *' -10:20: Cannot convert 'int *' to Python object +7:9: Cannot assign type 'char *' to 'int' +8:9: Cannot convert Python object to 'int *' +10:10: Cannot convert 'int *' to Python object """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_autotestdict.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_autotestdict.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_autotestdict.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_autotestdict.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -7,5 +7,5 @@ pass _ERRORS = u""" -6:0: The autotestdict compiler directive is not allowed in function scope +5:0: The autotestdict compiler directive is not allowed in function scope """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_binop_and.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_binop_and.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_binop_and.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_binop_and.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,14 @@ +# mode: error +# tag: and, binop, warnings + +def test_and(a, b): + return a && b + + +_WARNINGS = """ +5:13: Found the C operator '&&', did you mean the Python operator 'and'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_binop_or.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_binop_or.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_binop_or.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_binop_or.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,14 @@ +# mode: error +# tag: or, binop, warnings + +def test_or(a, b): + return a || b + + +_WARNINGS = """ +5:13: Found the C operator '||', did you mean the Python operator 'or'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_boolcoerce.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_boolcoerce.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_boolcoerce.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_boolcoerce.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -30,15 +30,15 @@ _ERRORS = u""" -7:26: 'struct_type_not_boolean' is not a constant, variable or function identifier -7:26: Type 'struct_type_not_boolean' not acceptable as a boolean +7:3: 'struct_type_not_boolean' is not a constant, variable or function identifier +7:3: Type 'struct_type_not_boolean' not acceptable as a boolean -14:21: 'struct_not_boolean' is not a constant, variable or function identifier -14:21: Type 'struct_not_boolean' not acceptable as a boolean +14:3: 'struct_not_boolean' is not a constant, variable or function identifier +14:3: Type 'struct_not_boolean' not acceptable as a boolean -21:25: 'union_type_not_boolean' is not a constant, variable or function identifier -21:25: Type 'union_type_not_boolean' not acceptable as a boolean +21:3: 'union_type_not_boolean' is not a constant, variable or function identifier +21:3: Type 'union_type_not_boolean' not acceptable as a boolean -28:20: 'union_not_boolean' is not a constant, variable or function identifier -28:20: Type 'union_not_boolean' not acceptable as a boolean +28:3: 'union_not_boolean' is not a constant, variable or function identifier +28:3: Type 'union_not_boolean' not acceptable as a boolean """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_bufaccess2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_bufaccess2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_bufaccess2.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_bufaccess2.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -32,5 +32,5 @@ 3: 9: 'nothing' is not a type identifier 24:11: Cannot access buffer with object dtype without gil 24:11: Assignment of Python object not allowed without gil -29:12: Assignment of Python object not allowed without gil +29:8: Assignment of Python object not allowed without gil """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cdefemptysue.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cdefemptysue.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cdefemptysue.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cdefemptysue.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,8 +8,21 @@ cdef enum ham: pass + + +cdef struct flat_spam: pass + +ctypedef union flat_eggs: pass + +cdef enum flat_ham: pass + + _ERRORS = u""" 3:5: Empty struct or union definition not allowed outside a 'cdef extern from' block 6:0: Empty struct or union definition not allowed outside a 'cdef extern from' block 9:5: Empty enum definition not allowed outside a 'cdef extern from' block + +13:5: Empty struct or union definition not allowed outside a 'cdef extern from' block +15:0: Empty struct or union definition not allowed outside a 'cdef extern from' block +17:5: Empty enum definition not allowed outside a 'cdef extern from' block """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cdef_in_py.py cython-0.20.1+1~202203241016-9537/tests/errors/e_cdef_in_py.py --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cdef_in_py.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cdef_in_py.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# mode: error + +def func(): + cdef int i + + +_ERRORS = """ +4:4: The 'cdef' keyword is only allowed in Cython files (pyx/pxi/pxd) +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cdef_keywords_T241.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cdef_keywords_T241.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cdef_keywords_T241.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cdef_keywords_T241.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 241 +# ticket: t241 # mode: error cdef some_function(x, y): @@ -69,35 +69,35 @@ _ERRORS = u""" -22:18: argument 'x' passed twice -23:18: argument 'x' passed twice -24:23: C function got unexpected keyword argument 'z' -25:18: C function got unexpected keyword argument 'z' -26:21: C function got unexpected keyword argument 'z' -27:25: C function got unexpected keyword argument 'z' -28:25: argument 'x' passed twice -29:25: argument 'x' passed twice -29:30: C function got unexpected keyword argument 'z' - -39:18: argument 'x' passed twice -40:21: argument 'x' passed twice -41:21: argument 'y' passed twice -42:21: argument 'x' passed twice -42:26: argument 'y' passed twice -43:21: argument 'y' passed twice -43:26: argument 'x' passed twice -44:23: argument 'x' passed twice -45:21: C function got unexpected keyword argument 'z' -46:23: C function got unexpected keyword argument 'z' -47:20: argument 'x' passed twice -48:20: argument 'x' passed twice -49:25: argument 'x' passed twice - -58:16: argument 's1' passed twice -59:26: argument 's1' passed twice -60:29: argument 's1' passed twice -61:29: argument 's2' passed twice +22:17: argument 'x' passed twice +23:17: argument 'x' passed twice +24:22: C function got unexpected keyword argument 'z' +25:17: C function got unexpected keyword argument 'z' +26:20: C function got unexpected keyword argument 'z' +27:24: C function got unexpected keyword argument 'z' +28:24: argument 'x' passed twice +29:24: argument 'x' passed twice +29:29: C function got unexpected keyword argument 'z' + +39:17: argument 'x' passed twice +40:20: argument 'x' passed twice +41:20: argument 'y' passed twice +42:20: argument 'x' passed twice +42:25: argument 'y' passed twice +43:20: argument 'y' passed twice +43:25: argument 'x' passed twice +44:22: argument 'x' passed twice +45:20: C function got unexpected keyword argument 'z' +46:22: C function got unexpected keyword argument 'z' +47:19: argument 'x' passed twice +48:19: argument 'x' passed twice +49:24: argument 'x' passed twice + +58:14: argument 's1' passed twice +59:24: argument 's1' passed twice +60:27: argument 's1' passed twice +61:27: argument 's2' passed twice -67:18: C function got unexpected keyword argument 'char' -68:28: C function got unexpected keyword argument 'char' +67:14: C function got unexpected keyword argument 'char' +68:24: C function got unexpected keyword argument 'char' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cenum.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cenum.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cenum.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cenum.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,5 +8,5 @@ a = 42 # assignment to non-lvalue _ERRORS = u""" -8:3: Assignment to non-lvalue 'a' +8:1: Assignment to non-lvalue 'a' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cenum_with_type.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cenum_with_type.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cenum_with_type.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cenum_with_type.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,8 @@ +# mode: error + +cdef enum Spam(int): + a, b + +_ERRORS = u""" +3:14: Expected ':', found '(' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cmethbasematch.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cmethbasematch.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cmethbasematch.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cmethbasematch.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,13 +0,0 @@ -# mode: error - -cdef class C: - cdef void f(self): - pass - -cdef class D(C): - cdef void f(self, int x): - pass -_ERRORS = u""" -8:6: Signature not compatible with previous declaration -4:6: Previous declaration is here -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cpp_only_features.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cpp_only_features.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cpp_only_features.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cpp_only_features.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,26 @@ +# mode: error +# tag: no-cpp, werror + +from cython.operator import typeid + +def use_typeid(): + cdef int i = 0 + print typeid(i) == typeid(i) + +cdef cppclass A: + pass + +def use_new(): + cdef A* x = new A() + +def use_del(): + cdef A a = A() + cdef A *p = &a + del p + +_ERRORS = """ +8:10: typeid operator only allowed in c++ +8:23: typeid operator only allowed in c++ +14:20: Operation only allowed in c++ +19:4: Operation only allowed in c++ +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cpp_references.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cpp_references.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cpp_references.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cpp_references.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,10 @@ +# mode: error +# tag: cpp, cpp11 + +cdef foo(object& x): pass +cdef bar(object&& x): pass + +_ERRORS=""" +4:15: Reference base type cannot be a Python object +5:15: Rvalue-reference base type cannot be a Python object +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cstruct.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cstruct.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cstruct.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cstruct.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -27,7 +27,7 @@ 7:39: C struct/union member cannot be a Python object 17:9: Object of type 'Spam' has no attribute 'k' 18:9: Cannot assign type 'float (*)[42]' to 'int' -19:24: Cannot assign type 'int' to 'float (*)[42]' +19:10: Cannot assign type 'int' to 'float (*)[42]' 22:10: Cannot select attribute of incomplete type 'Grail' 23:6: Cannot select attribute of incomplete type 'Grail' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cunion.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cunion.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cunion.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cunion.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -25,5 +25,5 @@ _ERRORS = """ -24:12: Cannot convert 'IllegalMix' to Python object +24:11: Cannot convert 'IllegalMix' to Python object """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_cython_parallel.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_cython_parallel.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_cython_parallel.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_cython_parallel.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -149,38 +149,39 @@ with cython.parallel.parallel(): pass + _ERRORS = u""" -e_cython_parallel.pyx:3:8: cython.parallel.parallel is not a module -e_cython_parallel.pyx:4:0: No such directive: cython.parallel.something -e_cython_parallel.pyx:6:7: cython.parallel.parallel is not a module -e_cython_parallel.pyx:7:0: No such directive: cython.parallel.something -e_cython_parallel.pyx:13:6: prange() can only be used as part of a for loop -e_cython_parallel.pyx:13:6: prange() can only be used without the GIL -e_cython_parallel.pyx:18:19: Invalid schedule argument to prange: invalid_schedule -c_cython_parallel.pyx:21:29: The parallel section may only be used without the GIL -e_cython_parallel.pyx:27:10: target may not be a Python object as we don't have the GIL -e_cython_parallel.pyx:30:9: Can only iterate over an iteration variable -e_cython_parallel.pyx:33:10: Must be of numeric type, not int * -e_cython_parallel.pyx:36:33: Nested parallel with blocks are disallowed -e_cython_parallel.pyx:39:12: The parallel directive must be called -e_cython_parallel.pyx:45:10: local variable 'y' referenced before assignment -e_cython_parallel.pyx:55:9: local variable 'y' referenced before assignment -e_cython_parallel.pyx:60:6: Reduction operator '*' is inconsistent with previous reduction operator '+' -e_cython_parallel.pyx:62:36: cython.parallel.parallel() does not take positional arguments -e_cython_parallel.pyx:65:36: Invalid keyword argument: invalid -e_cython_parallel.pyx:73:12: Yield not allowed in parallel sections -e_cython_parallel.pyx:77:16: Yield not allowed in parallel sections -e_cython_parallel.pyx:97:19: Cannot assign to private of outer parallel block -e_cython_parallel.pyx:98:19: Cannot assign to private of outer parallel block -e_cython_parallel.pyx:104:6: Reductions not allowed for parallel blocks -e_cython_parallel.pyx:110:7: local variable 'i' referenced before assignment -e_cython_parallel.pyx:119:17: Cannot read reduction variable in loop body -e_cython_parallel.pyx:121:20: stop argument must be numeric -e_cython_parallel.pyx:121:19: prange() can only be used without the GIL -e_cython_parallel.pyx:131:8: Memoryview slices can only be shared in parallel sections -e_cython_parallel.pyx:133:42: Must provide schedule with chunksize -e_cython_parallel.pyx:136:62: Chunksize must not be negative -e_cython_parallel.pyx:139:62: Chunksize not valid for the schedule runtime -e_cython_parallel.pyx:145:70: Calling gil-requiring function not allowed without gil -e_cython_parallel.pyx:149:33: Nested parallel with blocks are disallowed +3:8: cython.parallel.parallel is not a module +4:0: No such directive: cython.parallel.something +6:7: cython.parallel.parallel is not a module +7:0: No such directive: cython.parallel.something +13:6: prange() can only be used as part of a for loop +13:6: prange() can only be used without the GIL +18:19: Invalid schedule argument to prange: invalid_schedule +21:29: The parallel section may only be used without the GIL +27:8: target may not be a Python object as we don't have the GIL +30:9: Can only iterate over an iteration variable +33:8: Must be of numeric type, not int * +36:33: Nested parallel with blocks are disallowed +39:12: The parallel directive must be called +45:8: local variable 'y' referenced before assignment +55:8: local variable 'y' referenced before assignment +60:4: Reduction operator '*' is inconsistent with previous reduction operator '+' +62:36: cython.parallel.parallel() does not take positional arguments +65:36: Invalid keyword argument: invalid +73:12: 'yield' not allowed in parallel sections +77:16: 'yield' not allowed in parallel sections +97:8: Cannot assign to private of outer parallel block +98:8: Cannot assign to private of outer parallel block +104:4: Reductions not allowed for parallel blocks +110:6: local variable 'i' referenced before assignment +119:14: Cannot read reduction variable in loop body +121:19: prange() can only be used without the GIL +121:20: stop argument must be numeric +131:4: Memoryview slices can only be shared in parallel sections +133:42: Must provide schedule with chunksize +136:62: Chunksize must not be negative +139:62: Chunksize not valid for the schedule runtime +145:70: Calling gil-requiring function not allowed without gil +149:33: Nested parallel with blocks are disallowed """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_decorators.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_decorators.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_decorators.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_decorators.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,13 +1,12 @@ # mode: error -_ERRORS = u""" -4:4 Expected a newline after decorator -""" - - class A: pass @A().a def f(): pass + +_ERRORS = u""" +6:4: Expected a newline after decorator +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_del.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_del.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_del.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_del.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -25,9 +25,9 @@ _ERRORS = u""" 10:9: Cannot assign to or delete this -11:48: Deletion of non-Python, non-C++ object +11:8: Deletion of non-Python, non-C++ object 13:9: Deletion of non-Python, non-C++ object 14:9: Deletion of non-Python, non-C++ object -19:9: can not delete variable 'a' referenced in nested scope -23:5: Deletion of global C variable +19:8: can not delete variable 'a' referenced in nested scope +23:4: Deletion of global C variable """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_directives.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_directives.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_directives.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_directives.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,6 +1,6 @@ # mode: error -# cython: nonexistant = True +# cython: nonexistent = True # cython: boundscheck = true # cython: boundscheck = 9 diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_excvalfunctype.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_excvalfunctype.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_excvalfunctype.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_excvalfunctype.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,7 +8,9 @@ grail = spam # type mismatch spam = grail # type mismatch + + _ERRORS = u""" -9:28: Cannot assign type 'spamfunc' to 'grailfunc' -10:28: Cannot assign type 'grailfunc' to 'spamfunc' +9:8: Cannot assign type 'spamfunc' to 'grailfunc' +10:7: Cannot assign type 'grailfunc' to 'spamfunc' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_exttype_total_ordering.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_exttype_total_ordering.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_exttype_total_ordering.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_exttype_total_ordering.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,178 @@ +# mode: error +# tag: total_ordering, warnings + +cimport cython + + +# Test all combinations with not enough methods. + +@cython.total_ordering +cdef class ExtNoFuncs: + pass + +@cython.total_ordering +cdef class ExtGe: + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLe: + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLeGe: + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtGt: + def __gt__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtGe: + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtLe: + def __gt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtLeGe: + def __gt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLt: + def __lt__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGe: + def __lt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtLe: + def __lt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtLeGe: + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGt: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtGe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtLe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtLeGe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtNe: + def __ne__(self, other): + return False + +@cython.total_ordering +cdef class ExtEq: + def __eq__(self, other): + return False + +@cython.total_ordering +cdef class ExtEqNe: + def __eq__(self, other): + return False + + def __ne__(self, other): + return False + + +_WARNINGS = """ +10:5: total_ordering directive used, but no comparison and equality methods defined +14:5: total_ordering directive used, but no equality method defined +19:5: total_ordering directive used, but no equality method defined +24:5: total_ordering directive used, but no equality method defined +32:5: total_ordering directive used, but no equality method defined +37:5: total_ordering directive used, but no equality method defined +45:5: total_ordering directive used, but no equality method defined +53:5: total_ordering directive used, but no equality method defined +64:5: total_ordering directive used, but no equality method defined +69:5: total_ordering directive used, but no equality method defined +77:5: total_ordering directive used, but no equality method defined +85:5: total_ordering directive used, but no equality method defined +96:5: total_ordering directive used, but no equality method defined +104:5: total_ordering directive used, but no equality method defined +115:5: total_ordering directive used, but no equality method defined +126:5: total_ordering directive used, but no equality method defined +140:5: total_ordering directive used, but no comparison methods defined +145:5: total_ordering directive used, but no comparison methods defined +150:5: total_ordering directive used, but no comparison methods defined +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_fstring.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_fstring.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_fstring.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_fstring.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,23 @@ +# mode: error +# tag: fstring + +def incorrect_fstrings(x): + return [ + f"{x}{'\\'}'{x+1}", + f"""{}""", + f"{}", + f"{x!}", + f"{", + f"{{}}}", + ] + + +_ERRORS = """ +6:16: backslashes not allowed in f-strings +7:14: empty expression not allowed in f-string +8:12: empty expression not allowed in f-string +9:14: missing '}' in format string expression, found '!' +10:12: empty expression not allowed in f-string +10:12: missing '}' in format string expression +11:15: f-string: single '}' is not allowed +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_index.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_index.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_index.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_index.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,9 +8,11 @@ int1 = array1[ptr1] # error int1 = int2[int3] # error obj1 = obj2[ptr1] # error + + _ERRORS = u""" 7:14: Invalid index type 'float' 8:14: Invalid index type 'float *' 9:12: Attempting to index non-array type 'int' -10:17: Cannot convert 'float *' to Python object +10:13: Cannot convert 'float *' to Python object """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_int_literals_py2.py cython-0.20.1+1~202203241016-9537/tests/errors/e_int_literals_py2.py --- cython-0.20.1+1~201611251650-6686/tests/errors/e_int_literals_py2.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_int_literals_py2.py 2022-03-24 10:16:46.000000000 +0000 @@ -3,7 +3,7 @@ def int_literals(): a = 1L # ok - b = 10000000000000L # ok + b = 10000000000000L # ok c = 1UL d = 10000000000000UL e = 10000000000000LL diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_invalid_num_threads.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_invalid_num_threads.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_invalid_num_threads.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_invalid_num_threads.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -9,12 +9,18 @@ pass # invalid +with nogil, parallel(num_threads=None, num_threads=None): + pass + with nogil, parallel(num_threads=0): pass with nogil, parallel(num_threads=i): pass +with nogil, parallel(num_threads=2, num_threads=2): + pass + with nogil, parallel(num_threads=2): for i in prange(10, num_threads=2): pass @@ -28,7 +34,9 @@ pass _ERRORS = u""" -e_invalid_num_threads.pyx:12:20: argument to num_threads must be greater than 0 -e_invalid_num_threads.pyx:19:19: num_threads already declared in outer section -e_invalid_num_threads.pyx:23:19: num_threads must be declared in the parent parallel section +12:20: Duplicate keyword argument found: num_threads +15:20: argument to num_threads must be greater than 0 +21:20: Duplicate keyword argument found: num_threads +25:19: num_threads already declared in outer section +29:19: num_threads must be declared in the parent parallel section """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_multass.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_multass.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_multass.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_multass.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -4,6 +4,8 @@ cdef int int1, int2, int3 cdef int *ptr2 int1, int3, obj1a = int2, ptr2, obj1b # error + + _ERRORS = u""" -6:31: Cannot assign type 'int *' to 'int' +6:27: Cannot assign type 'int *' to 'int' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_nogilfunctype.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_nogilfunctype.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_nogilfunctype.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_nogilfunctype.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,18 @@ # mode: error +# tag: warnings cdef extern from *: - cdef void f() - cdef void (*fp)() nogil + cdef void f() + cdef void (*fp)() nogil + ctypedef void (*fp_t)() nogil fp = f +fp = f + _ERRORS = u""" -7:6: Cannot assign type 'void (void)' to 'void (*)(void) nogil' +9:5: Cannot assign type 'void (void)' to 'void (*)(void) nogil' +""" + +_WARNINGS = """ +10:5: Casting a GIL-requiring function into a nogil function circumvents GIL validation """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_nonlocal_T490.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_nonlocal_T490.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_nonlocal_T490.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_nonlocal_T490.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -30,7 +30,9 @@ _ERRORS = u""" 4:4: no binding for nonlocal 'no_such_name' found +10:8: Previous declaration is here 11:8: 'x' redeclared as nonlocal 16:4: no binding for nonlocal 'global_name' found +27:8: Previous declaration is here 28:8: 'x' redeclared as nonlocal """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_pure_cimports.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_pure_cimports.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_pure_cimports.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_pure_cimports.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,31 @@ +# mode: error +# tag: pure, import, cimport + +import cython.cimportsy # FIXME: not currently an error? + +import cython.cimports +import cython.cimports.libc +import cython.cimports as cim + +cimport cython.cimports +cimport cython.cimports.libc +cimport cython.cimports as cim +import cython.cimports.libc as cython + + +# ok +import cython.cimports.libc as libc +from cython.cimports import libc +from cython.cimports cimport libc + + +_ERRORS = """ +6:7: Cannot cimport the 'cython.cimports' package directly, only submodules. +7:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +8:7: Cannot cimport the 'cython.cimports' package directly, only submodules. +10:8: Cannot cimport the 'cython.cimports' package directly, only submodules. +11:8: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +12:8: Cannot cimport the 'cython.cimports' package directly, only submodules. +# The following is not an accurate error message, but it's difficult to distinguish this case. And it's rare. +13:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_return.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_return.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_return.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_return.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -9,8 +9,10 @@ cdef int *p return # error return p # error + + _ERRORS = u""" -6:17: Return with value in void function +6:8: Return with value in void function 10:1: Return value required -11:17: Cannot assign type 'int *' to 'int' +11:8: Cannot assign type 'int *' to 'int' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_slice.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_slice.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_slice.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_slice.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -17,10 +17,10 @@ pass _ERRORS = u""" -5:20: Cannot convert 'int *' to Python object -6:21: Cannot convert 'int *' to Python object -7:22: Cannot convert 'int *' to Python object -12:16: C array iteration requires known end index +5:16: Cannot convert 'int *' to Python object +6:17: Cannot convert 'int *' to Python object +7:18: Cannot convert 'int *' to Python object +12:9: C array iteration requires known end index 14:16: C array iteration requires known end index -16:22: C array iteration requires known step size and end index +16:21: C array iteration requires known step size and end index """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_switch.pyx cython-0.20.1+1~202203241016-9537/tests/errors/e_switch.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/e_switch.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_switch.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -10,5 +10,5 @@ print 34 _ERRORS = u""" -5:19: undeclared name not builtin: NONEXISTING +5:8: undeclared name not builtin: NONEXISTING """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_tuple_args_T692.py cython-0.20.1+1~202203241016-9537/tests/errors/e_tuple_args_T692.py --- cython-0.20.1+1~201611251650-6686/tests/errors/e_tuple_args_T692.py 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_tuple_args_T692.py 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 692 +# ticket: t692 # mode: error def func((a, b)): @@ -6,7 +6,6 @@ _ERRORS = u""" 4:9: Missing argument name -5:13: undeclared name not builtin: a -5:16: undeclared name not builtin: b +5:11: undeclared name not builtin: a +5:15: undeclared name not builtin: b """ - diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/e_typing_optional.py cython-0.20.1+1~202203241016-9537/tests/errors/e_typing_optional.py --- cython-0.20.1+1~201611251650-6686/tests/errors/e_typing_optional.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/e_typing_optional.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,34 @@ +# mode: error + +import cython + +try: + from typing import Optional +except ImportError: + pass + + +def optional_pytypes(i: Optional[int], f: Optional[float]): + pass + + +def optional_cython_types(i: Optional[cython.int], d: Optional[cython.double], f: Optional[cython.float]): + pass + + +MyStruct = cython.struct(a=cython.int, b=cython.double) + +def optional_cstruct(x: Optional[MyStruct]): + pass + + +_ERRORS = """ +15:29: Only Python type arguments can use typing.Optional[...] +15:54: Only Python type arguments can use typing.Optional[...] +15:82: Only Python type arguments can use typing.Optional[...] +21:24: Only Python type arguments can use typing.Optional[...] + +# FIXME: these should be allowed! +11:24: Only Python type arguments can use typing.Optional[...] +11:42: Only Python type arguments can use typing.Optional[...] +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/extended_unpacking.pyx cython-0.20.1+1~202203241016-9537/tests/errors/extended_unpacking.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/extended_unpacking.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/extended_unpacking.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -45,5 +45,5 @@ # types() 32:15: Cannot coerce list to type 'int' -33:10: starred target must have Python object (list) type +33:8: starred target must have Python object (list) type """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/fused_types.pyx cython-0.20.1+1~202203241016-9537/tests/errors/fused_types.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/fused_types.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/fused_types.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,5 @@ # mode: error +# ticket: 1772 cimport cython from cython import fused_type @@ -48,6 +49,18 @@ def inner(): cdef cython.floating g + +# Mixing const and non-const type makes fused type ambiguous +cdef fused mix_const_t: + int + const int + +cdef cdef_func_with_mix_const_type(mix_const_t val): + print(val) + +cdef_func_with_mix_const_type(1) + + # This is all valid dtype5 = fused_type(int, long, float) dtype6 = cython.fused_type(int, long) @@ -63,18 +76,40 @@ func(x, y) +cdef floating return_type_unfindable1(cython.integral x): + return 1.0 + +cpdef floating return_type_unfindable2(cython.integral x): + return 1.0 + +cdef void contents_unfindable1(cython.integral x): + z: floating = 1 # note: cdef variables also fail with an error but not by the time this test aborts + sz = sizeof(floating) + _ERRORS = u""" -10:15: fused_type does not take keyword arguments -15:38: Type specified multiple times -26:4: Invalid use of fused types, type cannot be specialized -26:4: Not enough types specified to specialize the function, int2_t is still fused -27:4: Invalid use of fused types, type cannot be specialized +11:15: fused_type does not take keyword arguments +16:33: Type specified multiple times +27:0: Invalid use of fused types, type cannot be specialized 27:4: Not enough types specified to specialize the function, int2_t is still fused -28:16: Call with wrong number of arguments (expected 2, got 1) -29:16: Call with wrong number of arguments (expected 2, got 3) -36:6: Invalid base type for memoryview slice: int * -39:0: Fused lambdas not allowed -42:5: Fused types not allowed here -45:9: Fused types not allowed here +28:0: Invalid use of fused types, type cannot be specialized +28:4: Not enough types specified to specialize the function, int2_t is still fused +29:16: Call with wrong number of arguments (expected 2, got 1) +30:16: Call with wrong number of arguments (expected 2, got 3) +37:6: Invalid base type for memoryview slice: int * +40:0: Fused lambdas not allowed +43:5: Fused types not allowed here +43:21: cdef variable 'x' declared after it is used +46:9: Fused types not allowed here +61:0: Invalid use of fused types, type cannot be specialized +61:29: ambiguous overloaded method +# Possibly duplicates the errors more often than we want +79:5: Return type is a fused type that cannot be determined from the function arguments +82:6: Return type is a fused type that cannot be determined from the function arguments +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/invalid_syntax_py.pyx cython-0.20.1+1~202203241016-9537/tests/errors/invalid_syntax_py.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/invalid_syntax_py.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/invalid_syntax_py.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,27 @@ +# mode: error +# tag: syntax + +def inline func() -> int: + pass + +def api func() -> int: + pass + +def nogil func() -> int: + pass + +def func() nogil: + pass + +def inline int* func(): + pass + + +_ERRORS = u""" +4:11: Cannot use cdef modifier 'inline' in Python function signature. Use a decorator instead. +7:8: Cannot use cdef modifier 'api' in Python function signature. Use a decorator instead. +10:10: Cannot use cdef modifier 'nogil' in Python function signature. Use a decorator instead. +13:11: Cannot use cdef modifier 'nogil' in Python function signature. Use a decorator instead. +16:11: Cannot use cdef modifier 'inline' in Python function signature. Use a decorator instead. +16:14: Expected '(', found '*'. Did you use cdef syntax in a Python declaration? Use decorators and Python type annotations instead. +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/literal_lists.pyx cython-0.20.1+1~202203241016-9537/tests/errors/literal_lists.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/literal_lists.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/literal_lists.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -9,5 +9,5 @@ return False _ERRORS = u""" -6:10: Literal list must be assigned to pointer at time of declaration +6:8: Literal list must be assigned to pointer at time of declaration """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/missing_baseclass_in_predecl_T262.pyx cython-0.20.1+1~202203241016-9537/tests/errors/missing_baseclass_in_predecl_T262.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/missing_baseclass_in_predecl_T262.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/missing_baseclass_in_predecl_T262.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 262 +# ticket: t262 # mode: error cdef class Album diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/missing_self_in_cpdef_method_T156.pyx cython-0.20.1+1~202203241016-9537/tests/errors/missing_self_in_cpdef_method_T156.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/missing_self_in_cpdef_method_T156.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/missing_self_in_cpdef_method_T156.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 156 +# ticket: t156 # mode: error cdef class B: diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/missing_self_in_cpdef_method_T165.pyx cython-0.20.1+1~202203241016-9537/tests/errors/missing_self_in_cpdef_method_T165.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/missing_self_in_cpdef_method_T165.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/missing_self_in_cpdef_method_T165.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 165 +# ticket: t165 # mode: error cdef class A: diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nogil_conditional.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nogil_conditional.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nogil_conditional.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nogil_conditional.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,81 @@ +# cython: remove_unreachable=False +# mode: error + +cdef int f_nogil(int x) nogil: + cdef int y + y = x + 10 + return y + + +def f_gil(x): + y = 0 + y = x + 100 + return y + + +def illegal_gil_usage(): + cdef int res = 0 + with nogil(True): + res = f_gil(res) + + with nogil(True): + res = f_gil(res) + + with gil(False): + res = f_gil(res) + + with nogil(False): + res = f_nogil(res) + + +def foo(a): + return a < 10 + + +def non_constant_condition(int x) -> int: + cdef int res = x + with nogil(x < 10): + res = f_nogil(res) + + with gil(foo(x)): + res = f_gil(res) + + +ctypedef fused number_or_object: + float + object + + +def fused_type(number_or_object x): + with nogil(number_or_object is object): + res = x + 1 + + # This should be fine + with nogil(number_or_object is float): + res = x + 1 + + return res + + +_ERRORS = u""" +19:14: Accessing Python global or builtin not allowed without gil +19:19: Calling gil-requiring function not allowed without gil +19:19: Coercion from Python not allowed without the GIL +19:19: Constructing Python tuple not allowed without gil +19:20: Converting to Python object not allowed without gil +21:13: Trying to release the GIL while it was previously released. +22:18: Accessing Python global or builtin not allowed without gil +22:23: Calling gil-requiring function not allowed without gil +22:23: Coercion from Python not allowed without the GIL +22:23: Constructing Python tuple not allowed without gil +22:24: Converting to Python object not allowed without gil +25:18: Accessing Python global or builtin not allowed without gil +25:23: Calling gil-requiring function not allowed without gil +25:23: Coercion from Python not allowed without the GIL +25:23: Constructing Python tuple not allowed without gil +25:24: Converting to Python object not allowed without gil +37:17: Non-constant condition in a `with nogil()` statement +40:16: Non-constant condition in a `with gil()` statement +51:8: Assignment of Python object not allowed without gil +51:16: Calling gil-requiring function not allowed without gil +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nogilfunctype.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nogilfunctype.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nogilfunctype.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nogilfunctype.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -12,5 +12,5 @@ fp = f _ERRORS = u""" -12:6: Cannot assign type 'void (void)' to 'void (*)(void) nogil' +12:5: Cannot assign type 'void (void)' to 'void (*)(void) nogil' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nogil.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nogil.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nogil.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nogil.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,14 +8,14 @@ cdef object z z = None -cdef void h(int x) nogil: +cdef void h(int x) nogil: # allowed p() cdef object p() nogil: pass -cdef void r() nogil: - q() +cdef void r() nogil: # allowed + q() # allowed cdef object m(): cdef object x, y = 0, obj @@ -23,11 +23,11 @@ global fred q() with nogil: - r() + r() # allowed to call plain C functions q() - i = 42 + i = 42 # allowed with type inference obj = None - 17L + 17L # allowed 7j help xxx = `"Hello"` @@ -45,17 +45,17 @@ {x, y} obj and x t(obj) -# f(42) # Cython handles this internally + f(42) x + obj -obj x = y = obj x, y = y, x obj[i] = x obj.fred = x - print obj + print obj # allowed! del fred return obj - raise obj + raise obj # allowed! if obj: pass while obj: @@ -90,29 +90,30 @@ with nogil: x -# For m(), the important thing is that there are errors on all lines in the range 23-69 -# except these: 29, 34, 44, 56, 58, 60, 62-64 +cdef int fstrings(int x, object obj) nogil except -1: + f"" # allowed + f"a" # allowed + f"a"f"b" # allowed + f"{x}" + f"{obj}" + _ERRORS = u""" 4:5: Function with Python return type cannot be declared nogil 7:5: Function declared nogil has Python locals or temporaries -9:6: Assignment of Python object not allowed without gil +9:4: Assignment of Python object not allowed without gil 12:5: Discarding owned Python object not allowed without gil 14:5: Function with Python return type cannot be declared nogil 18:5: Calling gil-requiring function not allowed without gil 27:9: Calling gil-requiring function not allowed without gil -29:12: Assignment of Python object not allowed without gil +29:8: Assignment of Python object not allowed without gil 31:16: Constructing complex number not allowed without gil -33:12: Assignment of Python object not allowed without gil +33:8: Assignment of Python object not allowed without gil 33:14: Backquote expression not allowed without gil -33:15: Operation not allowed without gil 34:15: Assignment of Python object not allowed without gil -34:15: Operation not allowed without gil 34:15: Python import not allowed without gil -35:8: Operation not allowed without gil 35:13: Python import not allowed without gil 35:25: Constructing Python list not allowed without gil -35:25: Operation not allowed without gil 36:17: Iterating over Python object not allowed without gil 38:11: Discarding owned Python object not allowed without gil 38:11: Indexing Python object not allowed without gil @@ -121,46 +122,51 @@ 40:11: Constructing Python slice object not allowed without gil 40:11: Discarding owned Python object not allowed without gil 40:11: Indexing Python object not allowed without gil -40:13: Converting to Python object not allowed without gil -40:15: Converting to Python object not allowed without gil -40:17: Converting to Python object not allowed without gil +40:12: Converting to Python object not allowed without gil +40:14: Converting to Python object not allowed without gil +40:16: Converting to Python object not allowed without gil 41:11: Accessing Python attribute not allowed without gil 41:11: Discarding owned Python object not allowed without gil 42:9: Constructing Python tuple not allowed without gil 42:9: Discarding owned Python object not allowed without gil 43:8: Constructing Python list not allowed without gil 43:8: Discarding owned Python object not allowed without gil -44:10: Constructing Python dict not allowed without gil -44:10: Discarding owned Python object not allowed without gil -45:10: Constructing Python set not allowed without gil -45:10: Discarding owned Python object not allowed without gil +44:9: Constructing Python dict not allowed without gil +44:9: Discarding owned Python object not allowed without gil +45:9: Constructing Python set not allowed without gil +45:9: Discarding owned Python object not allowed without gil 46:12: Discarding owned Python object not allowed without gil 46:12: Truth-testing Python object not allowed without gil -47:13: Python type test not allowed without gil +47:10: Python type test not allowed without gil +48:9: Discarding owned Python object not allowed without gil 49:10: Discarding owned Python object not allowed without gil 49:10: Operation not allowed without gil 50:8: Discarding owned Python object not allowed without gil 50:8: Operation not allowed without gil -51:10: Assignment of Python object not allowed without gil -51:14: Assignment of Python object not allowed without gil -52:9: Assignment of Python object not allowed without gil -52:13: Assignment of Python object not allowed without gil -52:16: Creating temporary Python reference not allowed without gil -52:19: Creating temporary Python reference not allowed without gil +51:8: Assignment of Python object not allowed without gil +51:12: Assignment of Python object not allowed without gil +52:8: Assignment of Python object not allowed without gil +52:11: Assignment of Python object not allowed without gil +52:15: Creating temporary Python reference not allowed without gil +52:18: Creating temporary Python reference not allowed without gil 53:11: Assignment of Python object not allowed without gil 53:11: Indexing Python object not allowed without gil 54:11: Accessing Python attribute not allowed without gil 54:11: Assignment of Python object not allowed without gil -55:8: Constructing Python tuple not allowed without gil -55:8: Python print statement not allowed without gil + 56:8: Deleting Python object not allowed without gil 57:8: Returning Python object not allowed without gil -58:8: Raising exception not allowed without gil -59:14: Truth-testing Python object not allowed without gil -61:17: Truth-testing Python object not allowed without gil + +59:11: Truth-testing Python object not allowed without gil +61:14: Truth-testing Python object not allowed without gil 63:8: For-loop using object bounds or target not allowed without gil -63:14: Coercion from Python not allowed without the GIL -63:25: Coercion from Python not allowed without the GIL +63:12: Coercion from Python not allowed without the GIL +63:24: Coercion from Python not allowed without the GIL 65:8: Try-except statement not allowed without gil 86:8: For-loop using object bounds or target not allowed without gil + +97:4: Discarding owned Python object not allowed without gil +97:6: String formatting not allowed without gil +98:4: Discarding owned Python object not allowed without gil +98:6: String formatting not allowed without gil """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_def.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_def.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_def.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_def.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -4,6 +4,6 @@ DEF ospath = os.path _ERRORS = u""" -4:15: Compile-time name 'os' not defined +4:13: Compile-time name 'os' not defined 4:15: Error in compile-time expression: AttributeError: 'NoneType' object has no attribute 'path' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_def_tuple.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_def_tuple.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_def_tuple.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_def_tuple.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,5 +8,5 @@ _ERRORS = u""" 5:32: Error in compile-time expression: IndexError: tuple index out of range -7:15: Invalid type for compile-time constant: [1, 2, 3] (type list) +7:4: Invalid type for compile-time constant: [1, 2, 3] (type list) """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_excval.pyx cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_excval.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/nonconst_excval.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/nonconst_excval.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +# mode: error + +import math + +cdef double cfunc(double x) except math.nan: + return x + + +_ERRORS = """ +5:39: Exception value must be constant +5:39: Not allowed in a constant expression +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/notcimportedT418.pyx cython-0.20.1+1~202203241016-9537/tests/errors/notcimportedT418.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/notcimportedT418.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/notcimportedT418.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 418 +# ticket: t418 # mode: error import somemod.child diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep487_exttype.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep487_exttype.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep487_exttype.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep487_exttype.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error + +cdef class Imp: + def __init_subclass__(cls, a=None, **kwargs): + super().__init_subclass__(**kwargs) + print(a) + +cdef class ExImp1(Imp): pass +class ExImp2(Imp, a=60): pass + +_ERRORS = u""" +4:4: '__init_subclass__' is not supported by extension class +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async10.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async10.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async10.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async10.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,26 +0,0 @@ -# mode: error -# tag: pep492, async - -async def genexpr(it): - return (await x for x in it) - - -async def listcomp(it): - return [await x for x in it] - - -async def setcomp(it): - return {await x for x in it} - - -async def dictcomp(it): - return {await x:x+1 for x in it} - - -# NOTE: CPython doesn't allow comprehensions either - - -_ERRORS = """ -5:12: 'await' not allowed in generators (use 'yield') -5:12: 'await' not supported here -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async2.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async2.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,11 +0,0 @@ -# mode: error -# tag: pep492, async - -async def foo(): - def foo(a:await list()): - pass - -_ERRORS = """ -5:14: 'await' not supported here -5:14: 'await' not supported here -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async3.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async3.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async3.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async3.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,9 +0,0 @@ -# mode: error -# tag: pep492, async - -async def foo(): - [i async for i in els] - -_ERRORS = """ -5:7: Expected ']', found 'async' -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async6.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async6.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async6.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async6.pyx 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -# mode: error -# tag: pep492, async - -async def foo(): - yield - -_ERRORS = """ -5:4: 'yield' not allowed in async coroutines (use 'await') -5:4: 'yield' not supported here -""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async7.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async7.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pep492_badsyntax_async7.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pep492_badsyntax_async7.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -6,5 +6,4 @@ _ERRORS = """ 5:4: 'yield from' not supported here -5:4: 'yield' not allowed in async coroutines (use 'await') """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/posonly2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/posonly2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/posonly2.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/posonly2.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(a, b/2, c): + pass + +_ERRORS = u""" +4:11: Syntax error in Python function argument list +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/posonly3.pyx cython-0.20.1+1~202203241016-9537/tests/errors/posonly3.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/posonly3.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/posonly3.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# mode: error +# tag: posonly + +def f(*args, /): + pass + +def f(*args, a, /): + pass + + +_ERRORS = u""" +4:13: Expected ')', found '/' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/posonly4.pyx cython-0.20.1+1~202203241016-9537/tests/errors/posonly4.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/posonly4.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/posonly4.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(/, a = 1): + pass + +_ERRORS = u""" +4:6: Got zero positional-only arguments despite presence of positional-only specifier '/' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/posonly5.pyx cython-0.20.1+1~202203241016-9537/tests/errors/posonly5.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/posonly5.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/posonly5.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,11 @@ +# mode: error +# tag: posonly + +def test_multiple_seps(a,/,b,/): + pass + +_ERRORS = u""" +4:29: Expected ')', found '/' +""" + + diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/posonly.pyx cython-0.20.1+1~202203241016-9537/tests/errors/posonly.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/posonly.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/posonly.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,102 @@ +# mode: error +# tag: posonly + +def f(a, b = 5, /, c): + pass + +def f(a = 5, b, /, c): + pass + +def f(a = 5, b, /): + pass + +def f(a, /, a): + pass + +def f(a, /, *, a): + pass + +#def f(a, b/2, c): #D +# pass + +#def f(*args, /): #D +# pass + +#def f(*args, a, /): +# pass + +#def f(**kwargs, /): +# pass + +#def f(/, a = 1): # D +# pass + +#def f(/, a): +# pass + +#def f(/): +# pass + +#def f(*, a, /): +# pass + +#def f(*, /, a): +# pass + +#def f(a, /, c, /): +# pass + +#def f(a, /, c, /, d): +# pass + +#def f(a, /, c, /, d, *, e): +# pass + +#def f(a, *, c, /, d, e): +# pass + +def test_invalid_syntax_lambda(self): + lambda a, b = 5, /, c: None + lambda a = 5, b, /, c: None + lambda a = 5, b, /: None + lambda a, /, a: None + lambda a, /, *, a: None +# lambda *args, /: None +# lambda *args, a, /: None +# lambda **kwargs, /: None +# lambda /, a = 1: None +# lambda /, a: None +# lambda /: None +# lambda *, a, /: None +# lambda *, /, a: None + +async def f(a, b = 5, /, c): + pass + +#def test_multiple_seps(a,/,b,/): +# pass + +_ERRORS = u""" +4:19: Non-default argument following default argument +7:13: Non-default argument following default argument +7:19: Non-default argument following default argument +10:13: Non-default argument following default argument +13:6: Previous declaration is here +13:12: 'a' redeclared +16:6: Previous declaration is here +16:15: 'a' redeclared +59:24: Non-default argument following default argument +60:18: Non-default argument following default argument +60:24: Non-default argument following default argument +61:18: Non-default argument following default argument +62:11: Previous declaration is here +62:17: 'a' redeclared +63:11: Previous declaration is here +63:20: 'a' redeclared +73:25: Non-default argument following default argument +""" + + + + + diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pure_errors.py cython-0.20.1+1~202203241016-9537/tests/errors/pure_errors.py --- cython-0.20.1+1~201611251650-6686/tests/errors/pure_errors.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pure_errors.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,64 @@ +# mode: error +# tag: warnings + +import cython + +@cython.cfunc +@cython.locals(x=cython.int) +@cython.returns(cython.int) +def cdef_needs_gil(x): + return x + 1 + + +@cython.cfunc +@cython.nogil +@cython.locals(x=cython.int) +@cython.returns(cython.int) +def cdef_nogil(x): + return x + 1 + + +@cython.cfunc +@cython.nogil(True) +@cython.locals(x=cython.int) +@cython.returns(cython.int) +def cdef_nogil_true(x): + return x + 1 + + +@cython.cfunc +@cython.nogil(False) +@cython.locals(x=cython.int) +@cython.returns(cython.int) +def cdef_nogil_false(x): + return x + 1 + + +@cython.locals(x=cython.int) +def test_cdef_nogil(x): + cdef_nogil(x) # ok + cdef_nogil_false(x) # ok + cdef_nogil_true(x) # ok + with cython.nogil: + cdef_nogil_true(x) # ok + cdef_needs_gil(x) # not ok + cdef_nogil_false(x) # not ok + + +@cython.nogil +def pyfunc(x): # invalid + return x + 1 + + +@cython.exceptval(-1) +@cython.cfunc +def test_cdef_return_object_broken(x: object) -> object: + return x + + +_ERRORS = """ +44:22: Calling gil-requiring function not allowed without gil +45:24: Calling gil-requiring function not allowed without gil +48:0: Python functions cannot be declared 'nogil' +53:0: Exception clause not allowed for function returning Python object +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pxd_cdef_class_declaration_T286.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pxd_cdef_class_declaration_T286.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pxd_cdef_class_declaration_T286.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pxd_cdef_class_declaration_T286.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 286 +# ticket: t286 # mode: error cdef class A: diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pxd_signature_mismatch.pxd cython-0.20.1+1~202203241016-9537/tests/errors/pxd_signature_mismatch.pxd --- cython-0.20.1+1~201611251650-6686/tests/errors/pxd_signature_mismatch.pxd 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pxd_signature_mismatch.pxd 2022-03-24 10:16:46.000000000 +0000 @@ -1,14 +1,35 @@ +# OK + +cdef int wider_exception_check(int x, int y) except? 0 + +cdef int no_exception_raised(int x, int y) except * + +cdef int any_exception_value1(int x, int y) except * + +cdef int any_exception_value2(int x, int y) except * + +cdef int any_exception_value3(int x, int y) except * + +cdef int any_exception_value4(int x, int y) except * + +cdef int optimised_exception_value(int x, int y) except? -1 + +# NOK cdef int wrong_args(int x, long y) cdef long wrong_return_type(int x, int y) -cdef int wrong_exception_check(int x, int y) except 0 +cdef int foreign_exception_value(int x, int y) except 0 + +cdef int narrower_exception_check(int x, int y) except 0 cdef int wrong_exception_value(int x, int y) except 0 cdef int wrong_exception_value_check(int x, int y) except 0 -cdef int inherit_exception_value(int x, int y) except 0 +cdef int wrong_exception_value_optimised_check(int x, int y) except? -2 + +cdef int wrong_exception_value_optimised(int x, int y) except -2 -cdef int inherit_exception_check(int x, int y) except * +cdef int narrower_exception_check_optimised(int x, int y) except -1 diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pxd_signature_mismatch.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pxd_signature_mismatch.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pxd_signature_mismatch.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pxd_signature_mismatch.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,13 +1,42 @@ # mode: error # tag: pxd +# OK + +cdef int wider_exception_check(int x, int y) except 0: + return 2 + +cdef int no_exception_raised(int x, int y): + return 2 + +cdef int any_exception_value1(int x, int y) except *: + return 2 + +cdef int any_exception_value2(int x, int y) except -1: + return 2 + +cdef int any_exception_value3(int x, int y) except -2: + return 2 + +cdef int any_exception_value4(int x, int y) except? -2: + return 2 + +cdef int optimised_exception_value(int x, int y) except *: # => except? -1 + return 2 + + +# NOK + cdef int wrong_args(int x, int y): return 2 cdef int wrong_return_type(int x, int y): return 2 -cdef int wrong_exception_check(int x, int y) except? 0: +cdef int foreign_exception_value(int x, int y): + return 2 + +cdef int narrower_exception_check(int x, int y) except? 0: return 2 cdef int wrong_exception_value(int x, int y) except 1: @@ -16,19 +45,24 @@ cdef int wrong_exception_value_check(int x, int y) except? 1: return 2 -cdef int inherit_exception_value(int x, int y): +cdef int wrong_exception_value_optimised_check(int x, int y) except *: + return 2 + +cdef int wrong_exception_value_optimised(int x, int y) except *: return 2 -cdef int inherit_exception_check(int x, int y): +cdef int narrower_exception_check_optimised(int x, int y) except *: return 2 _ERRORS = """ -4:5: Function signature does not match previous declaration -7:5: Function signature does not match previous declaration -10:5: Function signature does not match previous declaration -13:5: Function signature does not match previous declaration -16:5: Function signature does not match previous declaration -19:5: Function signature does not match previous declaration -22:5: Function signature does not match previous declaration +30:5: Function signature does not match previous declaration +33:5: Function signature does not match previous declaration +36:5: Function signature does not match previous declaration +39:5: Function signature does not match previous declaration +42:5: Function signature does not match previous declaration +45:5: Function signature does not match previous declaration +48:5: Function signature does not match previous declaration +51:5: Function signature does not match previous declaration +54:5: Function signature does not match previous declaration """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/pyobjcastdisallow_T313.pyx cython-0.20.1+1~202203241016-9537/tests/errors/pyobjcastdisallow_T313.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/pyobjcastdisallow_T313.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/pyobjcastdisallow_T313.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 313 +# ticket: t313 # mode: error a = 3 diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/redeclaration_of_var_by_cfunc_T600.pyx cython-0.20.1+1~202203241016-9537/tests/errors/redeclaration_of_var_by_cfunc_T600.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/redeclaration_of_var_by_cfunc_T600.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/redeclaration_of_var_by_cfunc_T600.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,14 @@ +# ticket: 600 +# mode: error + +cdef class Bar: + cdef list _operands + + cdef int _operands(self): + return -1 + + +_ERRORS = """ +7:9: '_operands' redeclared +5:14: Previous declaration is here +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/return_outside_function_T135.pyx cython-0.20.1+1~202203241016-9537/tests/errors/return_outside_function_T135.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/return_outside_function_T135.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/return_outside_function_T135.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,5 @@ # cython: remove_unreachable=False -# ticket: 135 +# ticket: t135 # mode: error def _runtime_True(): diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/reversed_literal_pyobjs.pyx cython-0.20.1+1~202203241016-9537/tests/errors/reversed_literal_pyobjs.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/reversed_literal_pyobjs.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/reversed_literal_pyobjs.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -10,6 +10,7 @@ pass for i in reversed(range(j, [], -2)): pass +# code below is no longer a compile-time error (although won't run without an exception) for i in reversed(range({}, j, 2)): pass for i in reversed(range({}, j, -2)): @@ -24,8 +25,4 @@ 7:24: Cannot coerce list to type 'long' 9:27: Cannot coerce list to type 'long' 11:27: Cannot coerce list to type 'long' -13:24: Cannot interpret dict as type 'long' -15:24: Cannot interpret dict as type 'long' -17:27: Cannot interpret dict as type 'long' -19:27: Cannot interpret dict as type 'long' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/string_assignments.pyx cython-0.20.1+1~202203241016-9537/tests/errors/string_assignments.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/string_assignments.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/string_assignments.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -76,39 +76,39 @@ _ERRORS = u""" 36:20: Unicode literals do not support coercion to C types other than Py_UNICODE/Py_UCS4 (for characters) or Py_UNICODE* (for strings). -37:22: Unicode objects only support coercion to Py_UNICODE*. -38:22: 'str' objects do not support coercion to C types (use 'bytes'?). +37:20: Unicode objects only support coercion to Py_UNICODE*. +38:20: 'str' objects do not support coercion to C types (use 'bytes'?). -40:27: Cannot assign type 'char *' to 'Py_UNICODE *' -41:27: Cannot convert 'bytes' object to Py_UNICODE*, use 'unicode'. -42:27: 'str' objects do not support coercion to C types (use 'unicode'?). +40:25: Cannot assign type 'char *' to 'Py_UNICODE *' +41:25: Cannot convert 'bytes' object to Py_UNICODE*, use 'unicode'. +42:25: 'str' objects do not support coercion to C types (use 'unicode'?). 43:25: Cannot convert 'bytes' object to Py_UNICODE*, use 'unicode'. 45:20: Cannot convert Unicode string to 'bytes' implicitly, encoding required. -46:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required. -47:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable. -48:23: Cannot convert 'basestring' object to bytes implicitly. This is not portable. +46:20: Cannot convert Unicode string to 'bytes' implicitly, encoding required. +47:20: Cannot convert 'str' to 'bytes' implicitly. This is not portable. +48:20: Cannot convert 'basestring' object to bytes implicitly. This is not portable. 50:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3. -51:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3. +51:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3. 52:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding. -53:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding. +53:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding. 55:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'') -56:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'') +56:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'') 57:20: Cannot convert 'bytes' object to unicode implicitly, decoding required -58:22: Cannot convert 'bytes' object to unicode implicitly, decoding required -59:22: Cannot convert 'char*' to unicode implicitly, decoding required +58:20: Cannot convert 'bytes' object to unicode implicitly, decoding required +59:20: Cannot convert 'char*' to unicode implicitly, decoding required 61:24: Cannot convert 'bytes' object to basestring implicitly. This is not portable to Py3. -62:26: Cannot convert 'bytes' object to basestring implicitly. This is not portable to Py3. +62:24: Cannot convert 'bytes' object to basestring implicitly. This is not portable to Py3. 64:19: Cannot assign type 'str object' to 'tuple object' 65:18: Cannot assign type 'unicode object' to 'tuple object' 66:18: Cannot assign type 'bytes object' to 'tuple object' -72:13: default encoding required for conversion from 'char *' to 'str object' +72:11: default encoding required for conversion from 'char *' to 'str object' 73:13: default encoding required for conversion from 'char *' to 'str object' -74:17: Cannot convert 'char*' to unicode implicitly, decoding required +74:15: Cannot convert 'char*' to unicode implicitly, decoding required 75:17: default encoding required for conversion from 'char *' to 'unicode object' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/subtyping_final_class.pyx cython-0.20.1+1~202203241016-9537/tests/errors/subtyping_final_class.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/subtyping_final_class.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/subtyping_final_class.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -10,5 +10,5 @@ pass _ERRORS = """ -9:5: Base class 'FinalClass' of type 'SubType' is final +9:19: Base class 'FinalClass' of type 'SubType' is final """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/tree_assert.pyx cython-0.20.1+1~202203241016-9537/tests/errors/tree_assert.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/tree_assert.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/tree_assert.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -11,8 +11,8 @@ _ERRORS = u""" -9:0: Expected path '//ComprehensionNode' not found in result tree -9:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree -9:0: Unexpected path '//NameNode' found in result tree -9:0: Unexpected path '//SimpleCallNode' found in result tree +5:0: Expected path '//ComprehensionNode' not found in result tree +5:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree +5:0: Unexpected path '//NameNode' found in result tree +5:0: Unexpected path '//SimpleCallNode' found in result tree """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/typoT304.pyx cython-0.20.1+1~202203241016-9537/tests/errors/typoT304.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/typoT304.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/typoT304.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,4 +1,4 @@ -# ticket: 304 +# ticket: t304 # mode: error def f(): diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/undefinedname.pyx cython-0.20.1+1~202203241016-9537/tests/errors/undefinedname.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/undefinedname.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/undefinedname.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -6,6 +6,6 @@ return _this_local_name_does_not_exist_ _ERRORS = u""" -3:37:undeclared name not builtin: _this_global_name_does_not_exist_ -6:43:undeclared name not builtin: _this_local_name_does_not_exist_ +3:4:undeclared name not builtin: _this_global_name_does_not_exist_ +6:11:undeclared name not builtin: _this_local_name_does_not_exist_ """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e1.pyx cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e1.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e1.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e1.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# mode: error + +★1 = 5 # invalid start symbol + +_ERRORS = u""" +4:0: Unrecognized character +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e2.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e2.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# mode: error + +class MyClass₡: # invalid continue symbol + pass + +_ERRORS = u""" +4:13: Unrecognized character +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e3.pyx cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e3.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e3.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e3.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# mode: error + +def f(): + a = 1 + ́b = 2 # looks like an indentation error but is actually a combining accent as the first letter of column 4 + c = 3 + +_ERRORS = u""" +6:4: Unrecognized character +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e4.pyx cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e4.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/unicode_identifiers_e4.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/unicode_identifiers_e4.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# mode: error + +cdef class C: + # these two symbols "\u1e69" and "\u1e9b\u0323" normalize to the same thing + # so the two attributes can't coexist + cdef int ṩomething + cdef double ẛ̣omething + +_ERRORS = u""" +7:13: Previous declaration is here +8:16: 'ṩomething' redeclared +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/uninitialized_lhs.pyx cython-0.20.1+1~202203241016-9537/tests/errors/uninitialized_lhs.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/uninitialized_lhs.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/uninitialized_lhs.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,7 +1,7 @@ # cython: warn.maybe_uninitialized=True # mode: error # tag: werror -# ticket: 739 +# ticket: t739 def index_lhs(a): cdef object idx @@ -12,6 +12,6 @@ a[:idx] = 1 _ERRORS = """ -8:9: local variable 'idx' referenced before assignment -12:10: local variable 'idx' referenced before assignment +8:6: local variable 'idx' referenced before assignment +12:7: local variable 'idx' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_numpy_arr_as_cppvec_ref.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_numpy_arr_as_cppvec_ref.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_numpy_arr_as_cppvec_ref.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_numpy_arr_as_cppvec_ref.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,10 +1,12 @@ # mode: error -# tag: cpp, werror, numpy +# tag: cpp, werror, numpy, no-cpp-locals import numpy as np cimport numpy as np from libcpp.vector cimport vector +np.import_array() + cdef extern from *: void cpp_function_vector1(vector[int]) void cpp_function_vector2(vector[int] &) @@ -24,8 +26,8 @@ _ERRORS = """ -17:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -18:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:31: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:36: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +19:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +20:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:33: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_python_list_as_cppset_ref.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_python_list_as_cppset_ref.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_python_list_as_cppset_ref.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_python_list_as_cppset_ref.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -15,5 +15,5 @@ _ERRORS = """ -14:33: Cannot pass Python object as C++ data structure reference (set[int] &), will pass by copy. +14:22: Cannot pass Python object as C++ data structure reference (set[int] &), will pass by copy. """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_undeclared.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_undeclared.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_undeclared.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_undeclared.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,20 @@ +# cython: warn.undeclared=True +# mode: error +# tag: werror + +def foo(): + a = 1 + return a + +cdef class Bar: + cdef int baz(self, a): + res = 0 + for i in range(3): + res += i + return res + +_ERRORS = """ +6:4: implicit declaration of 'a' +11:8: implicit declaration of 'res' +12:12: implicit declaration of 'i' +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_cpp.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_cpp.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_cpp.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_cpp.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +# cython: warn.maybe_uninitialized=True +# mode: error +# tag: cpp, werror + +from cython.operator import typeid + +def uninitialized_in_typeid(): + cdef int i + print typeid(i) == typeid(i) + +_ERRORS = """ +""" diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_del.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_del.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_del.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_del.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -9,6 +9,6 @@ return a, b _ERRORS = """ -7:12: local variable 'b' referenced before assignment -9:12: local variable 'a' referenced before assignment +7:11: local variable 'b' referenced before assignment +9:11: local variable 'a' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_exc.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_exc.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_exc.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_exc.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -113,13 +113,13 @@ print f _ERRORS = """ -12:12: local variable 'e' might be referenced before assignment -12:15: local variable 'i' might be referenced before assignment -19:12: local variable 'a' might be referenced before assignment -63:16: local variable 'x' might be referenced before assignment -69:16: local variable 'x' might be referenced before assignment -77:14: local variable 'oops' might be referenced before assignment -93:16: local variable 'x' might be referenced before assignment -101:16: local variable 'x' might be referenced before assignment -113:15: local variable 'f' might be referenced before assignment +12:11: local variable 'e' might be referenced before assignment +12:14: local variable 'i' might be referenced before assignment +19:11: local variable 'a' might be referenced before assignment +63:15: local variable 'x' might be referenced before assignment +69:15: local variable 'x' might be referenced before assignment +77:10: local variable 'oops' might be referenced before assignment +93:15: local variable 'x' might be referenced before assignment +101:15: local variable 'x' might be referenced before assignment +113:14: local variable 'f' might be referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_for.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_for.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_for.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_for.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -81,13 +81,13 @@ _ERRORS = """ -8:12: local variable 'a' might be referenced before assignment -14:12: local variable 'a' might be referenced before assignment -26:12: local variable 'i' might be referenced before assignment -31:12: local variable 'i' might be referenced before assignment -37:16: local variable 'x' might be referenced before assignment -44:11: local variable 'x' might be referenced before assignment -51:11: local variable 'x' might be referenced before assignment -58:19: local variable 'x' might be referenced before assignment -66:19: local variable 'x' might be referenced before assignment +8:11: local variable 'a' might be referenced before assignment +14:11: local variable 'a' might be referenced before assignment +26:11: local variable 'i' might be referenced before assignment +31:11: local variable 'i' might be referenced before assignment +37:15: local variable 'x' might be referenced before assignment +44:10: local variable 'x' might be referenced before assignment +51:10: local variable 'x' might be referenced before assignment +58:18: local variable 'x' might be referenced before assignment +66:18: local variable 'x' might be referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_generators.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_generators.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_generators.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_generators.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -8,5 +8,5 @@ x = i + i _ERRORS = """ -7:15: local variable 'x' might be referenced before assignment +7:14: local variable 'x' might be referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_py2.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_py2.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_py2.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_py2.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -18,9 +18,9 @@ _ERRORS = """ -7:12: local variable 'i' might be referenced before assignment -12:12: undeclared name not builtin: i -12:15: undeclared name not builtin: j -16:11: local variable 'i' referenced before assignment -16:14: local variable 'j' referenced before assignment +7:11: local variable 'i' might be referenced before assignment +12:11: undeclared name not builtin: i +12:14: undeclared name not builtin: j +16:10: local variable 'i' referenced before assignment +16:13: local variable 'j' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_py3.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_py3.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_py3.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_py3.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -19,6 +19,6 @@ _ERRORS = """ -10:9: local variable 'i' referenced before assignment -16:9: local variable 'i' referenced before assignment +10:8: local variable 'i' referenced before assignment +16:8: local variable 'i' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -112,22 +112,26 @@ args = [] kwargs = {} +def uninitialized_in_sizeof(): + cdef int i + print sizeof(i) + _ERRORS = """ -6:11: local variable 'a' referenced before assignment -12:12: local variable 'a' might be referenced before assignment -29:12: local variable 'a' might be referenced before assignment -35:15: local variable 'b' might be referenced before assignment -58:11: local variable 'a' referenced before assignment -58:14: local variable 'b' referenced before assignment -62:13: local variable 'bar' referenced before assignment -66:13: local variable 'foo' referenced before assignment -71:17: local variable 'exc' referenced before assignment -71:22: local variable 'msg' referenced before assignment +6:10: local variable 'a' referenced before assignment +12:11: local variable 'a' might be referenced before assignment +29:11: local variable 'a' might be referenced before assignment +35:14: local variable 'b' might be referenced before assignment +58:10: local variable 'a' referenced before assignment +58:13: local variable 'b' referenced before assignment +62:10: local variable 'bar' referenced before assignment +66:10: local variable 'foo' referenced before assignment +71:14: local variable 'exc' referenced before assignment +71:19: local variable 'msg' referenced before assignment 78:4: local variable 'decorator' referenced before assignment -85:23: local variable 'default' referenced before assignment -91:17: local variable 'bar' referenced before assignment +85:16: local variable 'default' referenced before assignment +91:14: local variable 'bar' referenced before assignment 97:4: local variable 'decorator' referenced before assignment -104:28: local variable 'Meta' referenced before assignment -110:19: local variable 'args' referenced before assignment -110:29: local variable 'kwargs' referenced before assignment +104:24: local variable 'Meta' referenced before assignment +110:15: local variable 'args' referenced before assignment +110:23: local variable 'kwargs' referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_while.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_while.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_while.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_while.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -55,8 +55,8 @@ _ERRORS = """ -9:12: local variable 'a' might be referenced before assignment -17:12: local variable 'a' might be referenced before assignment -32:19: local variable 'x' might be referenced before assignment -40:19: local variable 'x' might be referenced before assignment +9:11: local variable 'a' might be referenced before assignment +17:11: local variable 'a' might be referenced before assignment +32:18: local variable 'x' might be referenced before assignment +40:18: local variable 'x' might be referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_with.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_with.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_uninitialized_with.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_uninitialized_with.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -24,7 +24,7 @@ print f _ERRORS = """ -7:15: local variable 'a' referenced before assignment -11:11: local variable 'm2' referenced before assignment -24:15: local variable 'f' might be referenced before assignment +7:14: local variable 'a' referenced before assignment +11:9: local variable 'm2' referenced before assignment +24:14: local variable 'f' might be referenced before assignment """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/errors/w_unused.pyx cython-0.20.1+1~202203241016-9537/tests/errors/w_unused.pyx --- cython-0.20.1+1~201611251650-6686/tests/errors/w_unused.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/errors/w_unused.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -49,10 +49,10 @@ _ERRORS = """ -6:6: Unused entry 'a' -9:9: Unused entry 'b' +6:4: Unused entry 'a' +9:7: Unused entry 'b' 12:15: Unused argument 'arg' -16:6: Unused result in 'r' +16:4: Unused result in 'r' 21:4: Unused entry '_unused_one' 25:4: Unused entry 'Unused' 35:16: Unused entry 'foo' @@ -61,6 +61,6 @@ 38:28: Unused argument 'kwargs' 41:26: Unused argument 'c' 41:26: Unused entry 'c' -42:6: Unused entry 'x' -46:10: Unused entry 'y' +42:4: Unused entry 'x' +46:8: Unused entry 'y' """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/graal_bugs.txt cython-0.20.1+1~202203241016-9537/tests/graal_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/graal_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/graal_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,6 @@ +# This list is definitely not complete! + +cythonize_script_package # runs forever, probably multiprocessing issues +callingconvention # gets stuck and runs forever +different_package_names # gets stuck and runs forever +duplicate_utilitycode_from_pyx diff -Nru cython-0.20.1+1~201611251650-6686/tests/limited_api_bugs.txt cython-0.20.1+1~202203241016-9537/tests/limited_api_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/limited_api_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/limited_api_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,11 @@ +# This file contains tests corresponding to unresolved bugs using CPython's +# Limited API which will be skipped in the normal testing run. + +convolve2 +dict_animal +not_none +queue3 +test_queue +memoryview +memview +buffer diff -Nru cython-0.20.1+1~201611251650-6686/tests/macos_cpp_bugs.txt cython-0.20.1+1~202203241016-9537/tests/macos_cpp_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/macos_cpp_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/macos_cpp_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,14 @@ +complex_numbers_T305 +complex_numbers_c99_T398 +complex_numbers_cpp +complex_numbers_cxx_T398 +cpdef_extern_func +cpp_classes_def +cpp_smart_ptr +cpp_stl_conversion +cpp_stl_function +cpp_stl_optional +cpp_stl_algo_comparison_ops +cpp_stl_algo_permutation_ops +cpp_stl_algo_sorted_ranges_set_ops +cpp_stl_algo_sorted_ranges_other_ops diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/cfunc_convert_with_memoryview.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/cfunc_convert_with_memoryview.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/cfunc_convert_with_memoryview.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/cfunc_convert_with_memoryview.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,51 @@ +# mode: run +# tag: autowrap +# cython: always_allow_keywords=True + +cdef void memoryview_func_a(double [:] x): + x[0] = 1 + +cdef void memoryview_func_b(double [::1] x): + x[0] = 2 + +cdef void memoryview_func_c(int [:] x): + x[0] = 1 + +cdef void memoryview_func_d(int [:] x): + x[0] = 2 + +cdef void memoryview_func_e(int [:,::1] x): + x[0,0] = 4 + +cdef void memoryview_func_f(int [::1,:] x): + x[0,0] = 4 + + +def test_memview_wrapping(): + """ + We're mainly concerned that the code compiles without the names clashing + >>> test_memview_wrapping() + 1.0 + 2.0 + 1 + 2 + """ + cdef a = memoryview_func_a + cdef b = memoryview_func_b + cdef c = memoryview_func_c + cdef d = memoryview_func_d + cdef e = memoryview_func_e + cdef f = memoryview_func_f + cdef double[1] double_arr = [0] + cdef int[1] int_arr = [0] + + a( double_arr) + print(double_arr[0]) + b( double_arr) + print(double_arr[0]) + c( int_arr) + print(int_arr[0]) + d( int_arr) + print(int_arr[0]) + # don't call e and f because it's harder without needing extra dependencies + # it's mostly a compile test for them diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/contig_check.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/contig_check.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/contig_check.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/contig_check.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,68 @@ +# mode: run +# tag: numpy, memoryview + +import numpy as np + +# Cython used to forget the "is contig" helper functions when both are used. + + +def copy_fortran3(double[:, :, :] mat): + """ + >>> a = np.ones((1, 1, 1), dtype=np.float64) + >>> c = copy_fortran3(a) + >>> (a == c).all() + True + >>> a = np.ones((4, 6, 8), dtype=np.float64, order='F') + >>> c = copy_fortran3(a) + >>> (a == c).all() + True + >>> a = np.ones((4, 6, 8), dtype=np.float64, order='C') + >>> c = copy_fortran3(a) + >>> (a == c).all() + True + """ + cdef int x, y, z + + x, y, z = np.shape(mat) + + if 1 == x == y == z: + # C- or F- contiguous just means "contiguous". + if mat.is_c_contig() or mat.is_f_contig(): + return mat.base + else: + return np.asarray(mat.copy_fortran()) + elif mat.is_f_contig(): + return mat.base + else: + return np.asarray(mat.copy_fortran()) + + +def copy_fortran2(double[:, :] mat): + """ + >>> a = np.ones((1, 1), dtype=np.float64) + >>> c = copy_fortran2(a) + >>> (a == c).all() + True + >>> a = np.ones((4, 6), dtype=np.float64, order='F') + >>> c = copy_fortran2(a) + >>> (a == c).all() + True + >>> a = np.ones((4, 6), dtype=np.float64, order='C') + >>> c = copy_fortran2(a) + >>> (a == c).all() + True + """ + cdef int rows, cols + + rows, cols = np.shape(mat) + + if rows == 1 or cols == 1: + # C- or F- contiguous just means "contiguous". + if mat.is_c_contig() or mat.is_f_contig(): + return mat.base + else: + return np.asarray(mat.copy_fortran()) + elif mat.is_f_contig(): + return mat.base + else: + return np.asarray(mat.copy_fortran()) diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/cythonarray.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/cythonarray.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/cythonarray.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/cythonarray.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -2,11 +2,26 @@ from __future__ import unicode_literals +# these imports allow testing different ways to access [[cython.]view.]array() from cython.view cimport array from cython cimport view as v cimport cython as cy -include "cythonarrayutil.pxi" +include "../testsupport/cythonarrayutil.pxi" + + +def length(shape): + """ + >>> len(length((2,))) + 2 + >>> len(length((2,3))) + 2 + >>> len(length((5,3,2))) + 5 + """ + cdef array cvarray = array(shape=shape, itemsize=sizeof(int), format="i", mode='c') + assert len(cvarray) == shape[0] + return cvarray def contiguity(): @@ -20,7 +35,7 @@ 2 3 2 ''' - cdef v.array cvarray = v.array(shape=(2,3), itemsize=sizeof(int), format="i", mode='c') + cdef v.array cvarray = cy.view.array(shape=(2,3), itemsize=sizeof(int), format="i", mode='c') assert cvarray.len == 2*3*sizeof(int), (cvarray.len, 2*3*sizeof(int)) assert cvarray.itemsize == sizeof(int) print cvarray.strides[0], cvarray.strides[1] @@ -194,3 +209,80 @@ mslice = a print mslice[0, 0], mslice[1, 0], mslice[2, 5] + +class InheritFrom(v.array): + """ + Test is just to confirm it works, not to do anything meaningful with it + (Be aware that itemsize isn't necessarily right) + >>> inst = InheritFrom(shape=(3, 3, 3), itemsize=4, format="i") + """ + pass + + +def test_char_array_in_python_api(*shape): + """ + >>> import sys + >>> if sys.version_info[0] < 3: + ... def bytes(b): return memoryview(b).tobytes() # don't call str() + + >>> arr1d = test_char_array_in_python_api(10) + >>> print(bytes(arr1d).decode('ascii')) + xxxxxxxxxx + >>> len(bytes(arr1d)) + 10 + >>> arr2d = test_char_array_in_python_api(10, 2) + >>> print(bytes(arr2d).decode('ascii')) + xxxxxxxxxxxxxxxxxxxx + >>> len(bytes(arr2d)) + 20 + + # memoryview + >>> len(bytes(memoryview(arr1d))) + 10 + >>> bytes(memoryview(arr1d)) == bytes(arr1d) + True + >>> len(bytes(memoryview(arr2d))) + 20 + >>> bytes(memoryview(arr2d)) == bytes(arr2d) + True + + # BytesIO + >>> from io import BytesIO + >>> BytesIO(arr1d).read() == bytes(arr1d) + True + >>> BytesIO(arr2d).read() == bytes(arr2d) + True + + >>> b = BytesIO() + >>> print(b.write(arr1d)) + 10 + >>> b.getvalue() == bytes(arr1d) or b.getvalue() + True + + >>> b = BytesIO() + >>> print(b.write(arr2d)) + 20 + >>> b.getvalue() == bytes(arr2d) or b.getvalue() + True + + # BufferedWriter (uses PyBUF_SIMPLE, see https://github.com/cython/cython/issues/3775) + >>> from io import BufferedWriter + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr1d)) + 10 + >>> bw.flush() + >>> b.getvalue() == bytes(arr1d) + True + + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr2d)) + 20 + >>> bw.flush() + >>> b.getvalue() == bytes(arr2d) + True + """ + arr = array(shape=shape, itemsize=sizeof(char), format='c', mode='c') + arr[:] = b'x' + return arr diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/error_declarations.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/error_declarations.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/error_declarations.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/error_declarations.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -78,10 +78,10 @@ 18:22: Axis specification only allowed in the 'step' slot. 19:19: Fortran contiguous specifier must follow an indirect dimension 20:22: Invalid axis specification. -21:25: Invalid axis specification. +21:19: Invalid axis specification. 22:22: no expressions allowed in axis spec, only names and literals. -25:51: Memoryview 'object[::1, :]' not conformable to memoryview 'object[:, ::1]'. -28:36: Different base types for memoryviews (int, Python object) +25:37: Memoryview 'object[::1, :]' not conformable to memoryview 'object[:, ::1]'. +28:17: Different base types for memoryviews (int, Python object) 31:9: Dimension may not be contiguous 37:9: Only one direct contiguous axis may be specified. 38:9:Only dimensions 3 and 2 may be contiguous and direct diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_annotation_typing.py cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_annotation_typing.py --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_annotation_typing.py 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_annotation_typing.py 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,54 @@ +# mode: run +# tag: pep484, numpy, pure3.0 +##, warnings + +import cython +import numpy + + +def one_dim(a: cython.double[:]): + """ + >>> a = numpy.ones((10,), numpy.double) + >>> one_dim(a) + (2.0, 1) + """ + a[0] *= 2 + return a[0], a.ndim + + +def one_dim_ccontig(a: cython.double[::1]): + """ + >>> a = numpy.ones((10,), numpy.double) + >>> one_dim_ccontig(a) + (2.0, 1) + """ + a[0] *= 2 + return a[0], a.ndim + + +def two_dim(a: cython.double[:,:]): + """ + >>> a = numpy.ones((10, 10), numpy.double) + >>> two_dim(a) + (3.0, 1.0, 2) + """ + a[0,0] *= 3 + return a[0,0], a[0,1], a.ndim + + +@cython.nogil +@cython.cfunc +def _one_dim_nogil_cfunc(a: cython.double[:]) -> cython.double: + a[0] *= 2 + return a[0] + + +def one_dim_nogil_cfunc(a: cython.double[:]): + """ + >>> a = numpy.ones((10,), numpy.double) + >>> one_dim_nogil_cfunc(a) + 2.0 + """ + with cython.nogil: + result = _one_dim_nogil_cfunc(a) + return result diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryviewattrs.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryviewattrs.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryviewattrs.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryviewattrs.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,13 +1,6 @@ # mode: run # tag: numpy -__test__ = {} - -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func - - cimport cython from cython.view cimport array @@ -15,7 +8,6 @@ cimport numpy as np -@testcase def test_shape_stride_suboffset(): u''' >>> test_shape_stride_suboffset() @@ -49,7 +41,6 @@ print c_contig.suboffsets[0], c_contig.suboffsets[1], c_contig.suboffsets[2] -@testcase def test_copy_to(): u''' >>> test_copy_to() @@ -72,7 +63,6 @@ print ' '.join(str(to_data[i]) for i in range(2*2*2)) -@testcase def test_overlapping_copy(): """ >>> test_overlapping_copy() @@ -88,7 +78,6 @@ assert slice[i] == 10 - 1 - i -@testcase def test_copy_return_type(): """ >>> test_copy_return_type() @@ -103,7 +92,6 @@ print(f_contig[2, 2]) -@testcase def test_partly_overlapping(): """ >>> test_partly_overlapping() @@ -119,7 +107,6 @@ for i in range(5): assert slice2[i] == i + 4 -@testcase @cython.nonecheck(True) def test_nonecheck1(): u''' @@ -131,7 +118,6 @@ cdef int[:,:,:] uninitialized print uninitialized.is_c_contig() -@testcase @cython.nonecheck(True) def test_nonecheck2(): u''' @@ -143,7 +129,6 @@ cdef int[:,:,:] uninitialized print uninitialized.is_f_contig() -@testcase @cython.nonecheck(True) def test_nonecheck3(): u''' @@ -155,7 +140,6 @@ cdef int[:,:,:] uninitialized uninitialized.copy() -@testcase @cython.nonecheck(True) def test_nonecheck4(): u''' @@ -167,7 +151,6 @@ cdef int[:,:,:] uninitialized uninitialized.copy_fortran() -@testcase @cython.nonecheck(True) def test_nonecheck5(): u''' @@ -179,7 +162,6 @@ cdef int[:,:,:] uninitialized uninitialized._data -@testcase def test_copy_mismatch(): u''' >>> test_copy_mismatch() @@ -193,7 +175,6 @@ mv1[...] = mv2 -@testcase def test_is_contiguous(): u""" >>> test_is_contiguous() @@ -222,7 +203,6 @@ print 'strided', strided[::2].is_c_contig() -@testcase def call(): u''' >>> call() @@ -260,7 +240,11 @@ print (mv3._data)[0] , (mv2._data)[0] , (mv1._data)[0] -@testcase + assert len(mv1) == 3 + assert len(mv2) == 3 + assert len(mv3) == 3 + + def two_dee(): u''' >>> two_dee() @@ -272,6 +256,15 @@ cdef long[:,::1] mv1, mv2, mv3 cdef array arr = array((2,2), sizeof(long), 'l') + assert len(arr) == 2 + + try: + _ = len(mv1) + except UnboundLocalError: + pass + else: + assert False, "UnboundLocalError not raised for uninitialised memory view" + cdef long *arr_data arr_data = arr.data @@ -299,7 +292,6 @@ print (mv3._data)[0] , (mv3._data)[1] , (mv3._data)[2] , (mv3._data)[3] -@testcase def fort_two_dee(): u''' >>> fort_two_dee() diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_cache_builtins.srctree cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_cache_builtins.srctree --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_cache_builtins.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_cache_builtins.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,21 @@ +PYTHON setup.py build_ext --inplace + +################ setup.py ##################### + +from distutils.core import setup +from Cython.Build import cythonize +from Cython.Compiler import Options + +Options.cache_builtins = False + +setup( + ext_modules = cythonize("mview.pyx") +) + +############### mview.pyx ################ + +# https://github.com/cython/cython/issues/3406 +# Failure was at Cython compilation stage + +def f(double [::1] x): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_inline_pxd.srctree cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_inline_pxd.srctree --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_inline_pxd.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_inline_pxd.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,35 @@ +# ticket: 1415 + +# Utility code from an inline function in a pxd file was not +# correctly included in a pyx file that cimported it. +# Do not add more to this test - it is intentionally minimal +# to avoid including the utility code through other means + +PYTHON setup.py build_ext --inplace +PYTHON -c "import uses_inline; uses_inline.main()" + +######## setup.py ######## + +from distutils.core import setup +from Cython.Distutils import build_ext +from Cython.Distutils.extension import Extension + +setup( + ext_modules = [ + Extension("uses_inline", ["uses_inline.pyx"]), + ], + cmdclass={'build_ext': build_ext}, +) + +######## has_inline.pxd ######## + +from libc.stdlib cimport malloc +cdef inline double[::1] mview(size_t size): + return malloc(size * sizeof(double)) + +######## uses_inline.pyx ######## + +from has_inline cimport mview +def main(): + return mview(1) + diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_no_binding_T3613.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_no_binding_T3613.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_no_binding_T3613.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_no_binding_T3613.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,21 @@ +# mode: compile +# tag: memoryview + +# cython: binding=False + +# See GH 3613 - when memoryviews were compiled with binding off they ended up in an +# inconsistent state where different directives were applied at different stages +# of the pipeline + +import cython + +def f(double[:] a): + pass + +@cython.binding(False) +def g(double[:] a): + pass + +@cython.binding(True) +def h(double[:] a): + pass diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_pep484_typing.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_pep484_typing.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview_pep484_typing.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview_pep484_typing.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,96 @@ +# mode: run +# tag: pep484, memoryview + +cimport cython + +include "../buffers/mockbuffers.pxi" + + +def get_int_2d(mslice: cython.int[:, :], i: cython.int, j: cython.int) -> cython.int: + """ + >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> get_int_2d(C, 1, 1) + acquired C + released C + 4 + + Check negative indexing: + >>> get_int_2d(C, -1, 0) + acquired C + released C + 3 + >>> get_int_2d(C, -1, -2) + acquired C + released C + 4 + >>> get_int_2d(C, -2, -3) + acquired C + released C + 0 + + Out-of-bounds errors: + >>> get_int_2d(C, 2, 0) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + >>> get_int_2d(C, 0, -4) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 1) + """ + buf = mslice + return buf[i, j] + + +def set_int_2d(mslice: cython.int[:, :], i: cython.int, j: cython.int, value: cython.int) -> cython.int: + """ + Uses get_int_2d to read back the value afterwards. For pure + unit test, one should support reading in MockBuffer instead. + + >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> set_int_2d(C, 1, 1, 10) + acquired C + released C + >>> get_int_2d(C, 1, 1) + acquired C + released C + 10 + + Check negative indexing: + >>> set_int_2d(C, -1, 0, 3) + acquired C + released C + >>> get_int_2d(C, -1, 0) + acquired C + released C + 3 + + >>> set_int_2d(C, -1, -2, 8) + acquired C + released C + >>> get_int_2d(C, -1, -2) + acquired C + released C + 8 + + >>> set_int_2d(C, -2, -3, 9) + acquired C + released C + >>> get_int_2d(C, -2, -3) + acquired C + released C + 9 + + Out-of-bounds errors: + >>> set_int_2d(C, 2, 0, 19) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 0) + >>> set_int_2d(C, 0, -4, 19) + Traceback (most recent call last): + ... + IndexError: Out of bounds on buffer access (axis 1) + + """ + buf = mslice + buf[i, j] = value diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memoryview.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memoryview.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,7 @@ # mode: run +# Test declarations, behaviour and coercions of the memoryview type itself. + u''' >>> f() >>> g() @@ -14,6 +16,9 @@ from cpython.ref cimport Py_INCREF, Py_DECREF cimport cython +import array as pyarray +from libc.stdlib cimport malloc, free + cdef extern from "Python.h": cdef int PyBUF_C_CONTIGUOUS @@ -152,6 +157,7 @@ cdef int[:] mv3 mv1 = array((10,), itemsize=sizeof(int), format='i') mv2 = mv1 + mv1 = mv1 mv1 = mv2 mv3 = mv2 @@ -244,7 +250,7 @@ >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ccqii")) [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] """ - buf = mslice + cdef object buf = mslice print sorted([(k, int(v)) for k, v in buf[0].items()]) def nested_struct(NestedStruct[:] mslice): @@ -256,7 +262,7 @@ >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['x']['a'], d['x']['b'], d['y']['a'], d['y']['b'], d['z'] @@ -272,7 +278,7 @@ 1 2 """ - buf = mslice + cdef object buf = mslice print buf[0]['a'], buf[0]['b'] def nested_packed_struct(NestedPackedStruct[:] mslice): @@ -286,7 +292,7 @@ >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['a'], d['b'], d['sub']['a'], d['sub']['b'], d['c'] @@ -296,7 +302,7 @@ >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) -1j """ - buf = mslice + cdef object buf = mslice print buf[0] def complex_inplace(long double complex[:] mslice): @@ -304,7 +310,7 @@ >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)])) (1+1j) """ - buf = mslice + cdef object buf = mslice buf[0] = buf[0] + 1 + 2j print buf[0] @@ -315,7 +321,7 @@ >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) 0.0 -1.0 """ - buf = mslice + cdef object buf = mslice print buf[0]['real'], buf[0]['imag'] # @@ -353,7 +359,7 @@ ... IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice return buf[i, j] def set_int_2d(int[:, :] mslice, int i, int j, int value): @@ -406,11 +412,48 @@ IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice buf[i, j] = value # +# auto type inference +# (note that for most numeric types "might_overflow" stops the type inference from working well) +# +def type_infer(double[:, :] arg): + """ + >>> type_infer(DoubleMockBuffer(None, range(6), (2,3))) + double + double[:] + double[:] + double[:, :] + """ + a = arg[0,0] + print(cython.typeof(a)) + b = arg[0] + print(cython.typeof(b)) + c = arg[0,:] + print(cython.typeof(c)) + d = arg[:,:] + print(cython.typeof(d)) + +# +# Loop optimization +# +@cython.test_fail_if_path_exists("//CoerceToPyTypeNode") +def memview_iter(double[:, :] arg): + """ + memview_iter(DoubleMockBuffer("C", range(6), (2,3))) + True + """ + cdef double total = 0 + for mview1d in arg: + for val in mview1d: + total += val + if total == 15: + return True + +# # Test all kinds of indexing and flags # @@ -420,10 +463,10 @@ >>> writable(R) acquired R released R - >>> [str(x) for x in R.recieved_flags] # Py2/3 + >>> [str(x) for x in R.received_flags] # Py2/3 ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] """ - buf = mslice + cdef object buf = mslice buf[2, 2, 1] = 23 def strided(int[:] mslice): @@ -438,7 +481,7 @@ >>> A.release_ok True """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig(int[::1] mslice): @@ -447,18 +490,18 @@ >>> c_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig_2d(int[:, ::1] mslice): """ - Multi-dim has seperate implementation + Multi-dim has separate implementation >>> A = IntMockBuffer(None, range(12), shape=(3,4)) >>> c_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[1, 3] def f_contig(int[::1, :] mslice): @@ -467,7 +510,7 @@ >>> f_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[0, 1] def f_contig_2d(int[::1, :] mslice): @@ -478,7 +521,7 @@ >>> f_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[3, 1] def generic(int[::view.generic, ::view.generic] mslice1, @@ -549,7 +592,7 @@ ... ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -564,7 +607,7 @@ ... ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -579,7 +622,7 @@ ... ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -594,7 +637,7 @@ ... ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -609,7 +652,7 @@ ... ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -623,6 +666,8 @@ def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (x).ob_refcnt @@ -644,7 +689,7 @@ {4: 23} 2 [34, 3] 2 """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print repr(buf[i]), (buf[i]).ob_refcnt @@ -665,7 +710,7 @@ (2, 3) >>> decref(b) """ - buf = mslice + cdef object buf = mslice buf[idx] = obj def assign_temporary_to_object(object[:] mslice): @@ -692,9 +737,19 @@ >>> assign_to_object(A, 1, a) >>> decref(a) """ - buf = mslice + cdef object buf = mslice buf[1] = {3-2: 2+(2*4)-2} + +def test_pyview_of_memview(int[:] ints): + """ + >>> A = IntMockBuffer(None, [1, 2, 3]) + >>> len(test_pyview_of_memview(A)) + 3 + """ + return ints + + def test_generic_slicing(arg, indirect=False): """ Test simple slicing @@ -730,7 +785,7 @@ """ cdef int[::view.generic, ::view.generic, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -813,7 +868,7 @@ released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -841,7 +896,7 @@ released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[-5:, 1, 1::2] c = b[4:1:-1, ::-1] d = c[2, 1:2] @@ -1039,3 +1094,114 @@ cdef char[:] aview = a return max(1, aview[0]), min(5, aview[2]) + + +@cython.test_fail_if_path_exists( + '//MemoryViewSliceNode', +) +@cython.test_assert_path_exists( + '//MemoryViewIndexNode', +) +#@cython.boundscheck(False) # reduce C code clutter +def optimised_index_of_slice(int[:,:,:] arr, int x, int y, int z): + """ + >>> arr = IntMockBuffer("A", list(range(10*10*10)), shape=(10,10,10)) + >>> optimised_index_of_slice(arr, 2, 3, 4) + acquired A + (123, 123) + (223, 223) + (133, 133) + (124, 124) + (234, 234) + (123, 123) + (123, 123) + (123, 123) + (134, 134) + (134, 134) + (234, 234) + (234, 234) + (234, 234) + released A + """ + print(arr[1, 2, 3], arr[1][2][3]) + print(arr[x, 2, 3], arr[x][2][3]) + print(arr[1, y, 3], arr[1][y][3]) + print(arr[1, 2, z], arr[1][2][z]) + print(arr[x, y, z], arr[x][y][z]) + + print(arr[1, 2, 3], arr[:, 2][1][3]) + print(arr[1, 2, 3], arr[:, 2, :][1, 3]) + print(arr[1, 2, 3], arr[:, 2, 3][1]) + print(arr[1, y, z], arr[1, :][y][z]) + print(arr[1, y, z], arr[1, :][y, z]) + + print(arr[x, y, z], arr[x][:][:][y][:][:][z]) + print(arr[x, y, z], arr[:][x][:][y][:][:][z]) + print(arr[x, y, z], arr[:, :][x][:, :][y][:][z]) + + +def test_assign_from_byteslike(byteslike): + # Once https://python3statement.org/ is accepted, should be just + # >>> test_assign_from_byteslike(bytes(b'hello')) + # b'hello' + # ... + """ + >>> print(test_assign_from_byteslike(bytes(b'hello')).decode()) + hello + >>> print(test_assign_from_byteslike(bytearray(b'howdy')).decode()) + howdy + """ + # fails on Python 2.7- with + # TypeError: an integer is required + # >>> print(test_assign_from_byteslike(pyarray.array('B', b'aloha')).decode()) + # aloha + # fails on Python 2.6- with + # NameError: name 'memoryview' is not defined + # >>> print(test_assign_from_byteslike(memoryview(b'bye!!')).decode()) + # bye!! + + def assign(m): + m[:] = byteslike + + cdef void *buf + cdef unsigned char[:] mview + buf = malloc(5) + try: + mview = (buf) + assign(mview) + return (buf)[:5] + finally: + free(buf) + +def multiple_memoryview_def(double[:] a, double[:] b): + return a[0] + b[0] + +cpdef multiple_memoryview_cpdef(double[:] a, double[:] b): + return a[0] + b[0] + +cdef multiple_memoryview_cdef(double[:] a, double[:] b): + return a[0] + b[0] + +multiple_memoryview_cdef_wrapper = multiple_memoryview_cdef + +def test_conversion_failures(): + """ + What we're concerned with here is that we don't lose references if one + of several memoryview arguments fails to convert. + + >>> test_conversion_failures() + """ + imb = IntMockBuffer("", range(1), shape=(1,)) + dmb = DoubleMockBuffer("", range(1), shape=(1,)) + for first, second in [(imb, dmb), (dmb, imb)]: + for func in [multiple_memoryview_def, multiple_memoryview_cpdef, multiple_memoryview_cdef_wrapper]: + # note - using python call of "multiple_memoryview_cpdef" deliberately + imb_before = get_refcount(imb) + dmb_before = get_refcount(dmb) + try: + func(first, second) + except: + assert get_refcount(imb) == imb_before, "before %s after %s" % (imb_before, get_refcount(imb)) + assert get_refcount(dmb) == dmb_before, "before %s after %s" % (dmb_before, get_refcount(dmb)) + else: + assert False, "Conversion should fail!" diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/memslice.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/memslice.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/memslice.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/memslice.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,5 +1,7 @@ # mode: run +# Test the behaviour of memoryview slicing and indexing, contiguity, etc. + # Note: see also bufaccess.pyx from __future__ import unicode_literals @@ -12,6 +14,7 @@ from cython.view cimport array from cython.parallel cimport prange, parallel +from functools import wraps import gc import sys @@ -21,18 +24,8 @@ import builtins -__test__ = {} - def testcase(func): - doctest = func.__doc__ - if sys.version_info >= (3, 0): - _u = str - else: - _u = unicode - if not isinstance(doctest, _u): - doctest = doctest.decode('UTF-8') - __test__[func.__name__] = doctest - + @wraps(func) def wrapper(*args, **kwargs): gc.collect() result = func(*args, **kwargs) @@ -43,7 +36,7 @@ include "../buffers/mockbuffers.pxi" -include "cythonarrayutil.pxi" +include "../testsupport/cythonarrayutil.pxi" def _print_attributes(memview): print "shape: " + " ".join(map(str, memview.shape)) @@ -279,8 +272,8 @@ @testcase def tuple_buffer_assignment1(a, b): """ - >>> A = IntMockBuffer("A", range(6)) - >>> B = IntMockBuffer("B", range(6)) + >>> A = IntMockBuffer("A", range(6)) # , writable=False) + >>> B = IntMockBuffer("B", range(6)) # , writable=False) >>> tuple_buffer_assignment1(A, B) acquired A acquired B @@ -293,8 +286,8 @@ @testcase def tuple_buffer_assignment2(tup): """ - >>> A = IntMockBuffer("A", range(6)) - >>> B = IntMockBuffer("B", range(6)) + >>> A = IntMockBuffer("A", range(6)) # , writable=False) + >>> B = IntMockBuffer("B", range(6)) # , writable=False) >>> tuple_buffer_assignment2((A, B)) acquired A acquired B @@ -312,7 +305,7 @@ released A After release """ - cdef int[:] x = IntMockBuffer("A", range(10)) + cdef int[:] x = IntMockBuffer("A", range(10)) # , writable=False) del x print "After release" @@ -358,7 +351,7 @@ def get_int_2d_uintindex(int[:, :] buf, unsigned int i, unsigned int j): """ Unsigned indexing: - >>> C = IntMockBuffer("C", range(6), (2,3)) + >>> C = IntMockBuffer("C", range(6), (2,3)) # , writable=False) >>> get_int_2d_uintindex(C, 0, 0) acquired C released C @@ -422,6 +415,10 @@ ... IndexError: Out of bounds on buffer access (axis 1) + >>> C = IntMockBuffer("C", range(6), (2,3), writable=False) + >>> set_int_2d(C, -2, -3, 9) + Traceback (most recent call last): + BufferError: Writable buffer requested from read-only mock: FORMAT | ND | STRIDES | WRITABLE """ buf[i, j] = value @@ -588,11 +585,11 @@ @testcase def list_comprehension(int[:] buf, len): """ - >>> list_comprehension(IntMockBuffer(None, [1,2,3]), 3) + >>> list_comprehension(IntMockBuffer(None, [1,2,3]), 3) # , writable=False), 3) 1|2|3 """ cdef int i - print u"|".join([unicode(buf[i]) for i in range(len)]) + print "|".join([str(buf[i]) for i in range(len)]) @testcase @cython.wraparound(False) @@ -600,7 +597,7 @@ """ Again, the most interesting thing here is to inspect the C source. - >>> A = IntMockBuffer(None, range(4)) + >>> A = IntMockBuffer(None, range(4)) # , writable=False) >>> wraparound_directive(A, 2, -1) 5 >>> wraparound_directive(A, -1, 2) @@ -625,22 +622,22 @@ >>> writable(R) acquired R released R - >>> [str(x) for x in R.recieved_flags] # Py2/3 + >>> [str(x) for x in R.received_flags] # Py2/3 ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] """ cdef unsigned short int[:, :, :] buf = obj buf[2, 2, 1] = 23 @testcase -def strided(int[:] buf): +def strided(const int[:] buf): """ - >>> A = IntMockBuffer("A", range(4)) + >>> A = IntMockBuffer("A", range(4), writable=False) >>> strided(A) acquired A released A 2 - >>> [str(x) for x in A.recieved_flags] # Py2/3 - ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] + >>> [str(x) for x in A.received_flags] # Py2/3 + ['FORMAT', 'ND', 'STRIDES'] Check that the suboffsets were patched back prior to release. >>> A.release_ok @@ -649,25 +646,25 @@ return buf[2] @testcase -def c_contig(int[::1] buf): +def c_contig(const int[::1] buf): """ - >>> A = IntMockBuffer(None, range(4)) + >>> A = IntMockBuffer(None, range(4), writable=False) >>> c_contig(A) 2 - >>> [str(x) for x in A.recieved_flags] - ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS', 'WRITABLE'] + >>> [str(x) for x in A.received_flags] + ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS'] """ return buf[2] @testcase def c_contig_2d(int[:, ::1] buf): """ - Multi-dim has seperate implementation + Multi-dim has separate implementation - >>> A = IntMockBuffer(None, range(12), shape=(3,4)) + >>> A = IntMockBuffer(None, range(12), shape=(3,4)) # , writable=False) >>> c_contig_2d(A) 7 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'C_CONTIGUOUS', 'WRITABLE'] """ return buf[1, 3] @@ -675,10 +672,10 @@ @testcase def f_contig(int[::1, :] buf): """ - >>> A = IntMockBuffer(None, range(4), shape=(2, 2), strides=(1, 2)) + >>> A = IntMockBuffer(None, range(4), shape=(2, 2), strides=(1, 2)) # , writable=False) >>> f_contig(A) 2 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS', 'WRITABLE'] """ return buf[0, 1] @@ -688,10 +685,10 @@ """ Must set up strides manually to ensure Fortran ordering. - >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4)) + >>> A = IntMockBuffer(None, range(12), shape=(4,3), strides=(1, 4)) # , writable=False) >>> f_contig_2d(A) 7 - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'F_CONTIGUOUS', 'WRITABLE'] """ return buf[3, 1] @@ -711,9 +708,9 @@ 11 released A released B - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] - >>> [str(x) for x in B.recieved_flags] + >>> [str(x) for x in B.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] """ print buf1[1, 1] @@ -741,9 +738,9 @@ # 11 # released A # released B -# >>> [str(x) for x in A.recieved_flags] +# >>> [str(x) for x in A.received_flags] # ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] -# >>> [str(x) for x in B.recieved_flags] +# >>> [str(x) for x in B.received_flags] # ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] # """ # print buf1[1, 1] @@ -771,9 +768,9 @@ 11 released A released B - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] - >>> [str(x) for x in B.recieved_flags] + >>> [str(x) for x in B.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] """ print buf1[1, 1] @@ -802,9 +799,9 @@ 11 released A released B - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] - >>> [str(x) for x in B.recieved_flags] + >>> [str(x) for x in B.received_flags] ['FORMAT', 'INDIRECT', 'ND', 'STRIDES', 'WRITABLE'] """ print buf1[1, 1] @@ -827,7 +824,7 @@ @testcase def safe_get(int[:] buf, int idx): """ - >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) # , writable=False) Validate our testing buffer... >>> safe_get(A, 0) @@ -857,7 +854,7 @@ def unsafe_get(int[:] buf, int idx): """ Access outside of the area the buffer publishes. - >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) # , writable=False) >>> unsafe_get(A, -4) 4 >>> unsafe_get(A, -5) @@ -870,7 +867,7 @@ @testcase def mixed_get(int[:] buf, int unsafe_idx, int safe_idx): """ - >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) + >>> A = IntMockBuffer(None, range(10), shape=(3,), offset=5) # , writable=False) >>> mixed_get(A, -4, 0) (4, 5) >>> mixed_get(A, 0, -4) @@ -902,12 +899,12 @@ """ Strided: - >>> printbuf_int_2d(IntMockBuffer("A", range(6), (2,3)), (2,3)) + >>> printbuf_int_2d(IntMockBuffer("A", range(6), (2,3), writable=False), (2,3)) acquired A 0 1 2 END 3 4 5 END released A - >>> printbuf_int_2d(IntMockBuffer("A", range(100), (3,3), strides=(20,5)), (3,3)) + >>> printbuf_int_2d(IntMockBuffer("A", range(100), (3,3), strides=(20,5), writable=False), (3,3)) acquired A 0 5 10 END 20 25 30 END @@ -915,14 +912,14 @@ released A Indirect: - >>> printbuf_int_2d(IntMockBuffer("A", [[1,2],[3,4]]), (2,2)) + >>> printbuf_int_2d(IntMockBuffer("A", [[1,2],[3,4]], writable=False), (2,2)) acquired A 1 2 END 3 4 END released A """ # should make shape builtin - cdef int[::view.generic, ::view.generic] buf + cdef const int[::view.generic, ::view.generic] buf buf = o cdef int i, j for i in range(shape[0]): @@ -933,14 +930,14 @@ @testcase def printbuf_float(o, shape): """ - >>> printbuf_float(FloatMockBuffer("F", [1.0, 1.25, 0.75, 1.0]), (4,)) + >>> printbuf_float(FloatMockBuffer("F", [1.0, 1.25, 0.75, 1.0], writable=False), (4,)) acquired F 1.0 1.25 0.75 1.0 END released F """ # should make shape builtin - cdef float[:] buf + cdef const float[:] buf buf = o cdef int i, j for i in range(shape[0]): @@ -982,9 +979,9 @@ @testcase def printbuf_td_cy_int(td_cy_int[:] buf, shape): """ - >>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_cy_int(IntMockBuffer(None, range(3)), (3,)) # , writable=False), (3,)) 0 1 2 END - >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_cy_int(ShortMockBuffer(None, range(3)), (3,)) # , writable=False), (3,)) Traceback (most recent call last): ... ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short' @@ -997,9 +994,9 @@ @testcase def printbuf_td_h_short(td_h_short[:] buf, shape): """ - >>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_short(ShortMockBuffer(None, range(3)), (3,)) # , writable=False), (3,)) 0 1 2 END - >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_short(IntMockBuffer(None, range(3)), (3,)) # , writable=False), (3,)) Traceback (most recent call last): ... ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int' @@ -1010,14 +1007,14 @@ print 'END' @testcase -def printbuf_td_h_cy_short(td_h_cy_short[:] buf, shape): +def printbuf_td_h_cy_short(const td_h_cy_short[:] buf, shape): """ - >>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_cy_short(ShortMockBuffer(None, range(3), writable=False), (3,)) 0 1 2 END - >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_cy_short(IntMockBuffer(None, range(3), writable=False), (3,)) Traceback (most recent call last): ... - ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int' + ValueError: Buffer dtype mismatch, expected 'const td_h_cy_short' but got 'int' """ cdef int i for i in range(shape[0]): @@ -1025,14 +1022,14 @@ print 'END' @testcase -def printbuf_td_h_ushort(td_h_ushort[:] buf, shape): +def printbuf_td_h_ushort(const td_h_ushort[:] buf, shape): """ - >>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_ushort(UnsignedShortMockBuffer(None, range(3), writable=False), (3,)) 0 1 2 END - >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3)), (3,)) + >>> printbuf_td_h_ushort(ShortMockBuffer(None, range(3), writable=False), (3,)) Traceback (most recent call last): ... - ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short' + ValueError: Buffer dtype mismatch, expected 'const td_h_ushort' but got 'short' """ cdef int i for i in range(shape[0]): @@ -1040,14 +1037,14 @@ print 'END' @testcase -def printbuf_td_h_double(td_h_double[:] buf, shape): +def printbuf_td_h_double(const td_h_double[:] buf, shape): """ - >>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125]), (3,)) + >>> printbuf_td_h_double(DoubleMockBuffer(None, [0.25, 1, 3.125], writable=False), (3,)) 0.25 1.0 3.125 END - >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125]), (3,)) + >>> printbuf_td_h_double(FloatMockBuffer(None, [0.25, 1, 3.125], writable=False), (3,)) Traceback (most recent call last): ... - ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float' + ValueError: Buffer dtype mismatch, expected 'const td_h_double' but got 'float' """ cdef int i for i in range(shape[0]): @@ -1063,6 +1060,8 @@ def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (x).ob_refcnt @@ -1079,7 +1078,7 @@ >>> a, b, c = "globally_unique_string_23234123", {4:23}, [34,3] >>> get_refcount(a), get_refcount(b), get_refcount(c) (2, 2, 2) - >>> A = ObjectMockBuffer(None, [a, b, c]) + >>> A = ObjectMockBuffer(None, [a, b, c]) # , writable=False) >>> printbuf_object(A, (3,)) 'globally_unique_string_23234123' 2 {4: 23} 2 @@ -1145,11 +1144,11 @@ "strided" by defaults which should show up in the flags. - >>> A = IntStridedMockBuffer("A", range(10)) + >>> A = IntStridedMockBuffer("A", range(10)) # , writable=False) >>> bufdefaults1(A) acquired A released A - >>> [str(x) for x in A.recieved_flags] + >>> [str(x) for x in A.received_flags] ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] """ pass @@ -1168,6 +1167,18 @@ print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e @testcase +def const_struct(const MyStruct[:] buf): + """ + See also buffmt.pyx + + >>> const_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], writable=False)) + 1 2 3 4 5 + >>> const_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ccqii", writable=False)) + 1 2 3 4 5 + """ + print buf[0].a, buf[0].b, buf[0].c, buf[0].d, buf[0].e + +@testcase def nested_struct(NestedStruct[:] buf): """ See also buffmt.pyx @@ -1180,6 +1191,18 @@ print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z @testcase +def const_nested_struct(const NestedStruct[:] buf): + """ + See also buffmt.pyx + + >>> const_nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], writable=False)) + 1 2 3 4 5 + >>> const_nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i", writable=False)) + 1 2 3 4 5 + """ + print buf[0].x.a, buf[0].x.b, buf[0].y.a, buf[0].y.b, buf[0].z + +@testcase def packed_struct(PackedStruct[:] buf): """ See also buffmt.pyx @@ -1195,6 +1218,21 @@ print buf[0].a, buf[0].b @testcase +def const_packed_struct(const PackedStruct[:] buf): + """ + See also buffmt.pyx + + >>> const_packed_struct(PackedStructMockBuffer(None, [(1, 2)], writable=False)) + 1 2 + >>> const_packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c^i}", writable=False)) + 1 2 + >>> const_packed_struct(PackedStructMockBuffer(None, [(1, 2)], format="T{c=i}", writable=False)) + 1 2 + + """ + print buf[0].a, buf[0].b + +@testcase def nested_packed_struct(NestedPackedStruct[:] buf): """ See also buffmt.pyx @@ -1210,9 +1248,24 @@ @testcase +def const_nested_packed_struct(const NestedPackedStruct[:] buf): + """ + See also buffmt.pyx + + >>> const_nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], writable=False)) + 1 2 3 4 5 + >>> const_nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ci^ci@i", writable=False)) + 1 2 3 4 5 + >>> const_nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i", writable=False)) + 1 2 3 4 5 + """ + print buf[0].a, buf[0].b, buf[0].sub.a, buf[0].sub.b, buf[0].c + + +@testcase def complex_dtype(long double complex[:] buf): """ - >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) + >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) # , writable=False)) -1j """ print buf[0] @@ -1231,7 +1284,7 @@ """ Note that the format string is "Zg" rather than "2g", yet a struct is accessed. - >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) + >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) # , writable=False)) 0.0 -1.0 """ print buf[0].real, buf[0].imag @@ -1359,7 +1412,7 @@ def test_generic_slicing(arg, indirect=False): """ Test simple slicing - >>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11))) + >>> test_generic_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11))) # , writable=False)) acquired A 3 9 2 308 -11 1 @@ -1367,7 +1420,7 @@ released A Test direct slicing, negative slice oob in dim 2 - >>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3))) + >>> test_generic_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3))) # , writable=False)) acquired A 0 0 2 12 -3 1 @@ -1375,13 +1428,13 @@ released A Test indirect slicing - >>> test_generic_slicing(IntMockBuffer("A", shape_5_3_4_list, shape=(5, 3, 4)), indirect=True) + >>> test_generic_slicing(IntMockBuffer("A", shape_5_3_4_list, shape=(5, 3, 4)), indirect=True) # , writable=False), indirect=True) acquired A 2 0 2 0 1 -1 released A - >>> test_generic_slicing(IntMockBuffer("A", shape_9_14_21_list, shape=(9, 14, 21)), indirect=True) + >>> test_generic_slicing(IntMockBuffer("A", shape_9_14_21_list, shape=(9, 14, 21)), indirect=True) # , writable=False), indirect=True) acquired A 3 9 2 10 1 -1 @@ -1413,7 +1466,7 @@ def test_indirect_slicing(arg): """ Test indirect slicing - >>> test_indirect_slicing(IntMockBuffer("A", shape_5_3_4_list, shape=(5, 3, 4))) + >>> test_indirect_slicing(IntMockBuffer("A", shape_5_3_4_list, shape=(5, 3, 4))) # , writable=False)) acquired A 5 3 2 0 0 -1 @@ -1428,7 +1481,7 @@ 58 released A - >>> test_indirect_slicing(IntMockBuffer("A", shape_9_14_21_list, shape=(9, 14, 21))) + >>> test_indirect_slicing(IntMockBuffer("A", shape_9_14_21_list, shape=(9, 14, 21))) # , writable=False)) acquired A 5 14 3 0 16 -1 @@ -1528,7 +1581,7 @@ All dimensions preceding dimension 1 must be indexed and not sliced """ cdef int[:, ::view.indirect, :] a = TestIndexSlicingDirectIndirectDims() - a_obj = a + cdef object a_obj = a print a[1][2][3] print a[1, 2, 3] @@ -1553,7 +1606,7 @@ Fused types would be convenient to test this stuff! Test simple slicing - >>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11))) + >>> test_direct_slicing(IntMockBuffer("A", range(8 * 14 * 11), shape=(8, 14, 11))) # , writable=False)) acquired A 3 9 2 308 -11 1 @@ -1561,7 +1614,7 @@ released A Test direct slicing, negative slice oob in dim 2 - >>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3))) + >>> test_direct_slicing(IntMockBuffer("A", range(1 * 2 * 3), shape=(1, 2, 3))) # , writable=False)) acquired A 0 0 2 12 -3 1 @@ -1586,7 +1639,7 @@ @testcase def test_slicing_and_indexing(arg): """ - >>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5)) + >>> a = IntStridedMockBuffer("A", range(10 * 3 * 5), shape=(10, 3, 5)) # , writable=False) >>> test_slicing_and_indexing(a) acquired A 5 2 @@ -1622,7 +1675,7 @@ ... IndexError: Index out of bounds (axis 1) """ - cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9)) + cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9)) # , writable=False) print a[:, 20] @@ -1665,7 +1718,7 @@ ... IndexError: Index out of bounds (axis 0) """ - cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9)) + cdef int[:, :] a = IntMockBuffer("A", range(4 * 9), shape=(4, 9)) # , writable=False) with nogil: a[100, 9:] @@ -1677,7 +1730,7 @@ for j in range(b.shape[1]): b[i, j] = -b[i, j] - return 1 + return len(a) @testcase def test_nogil(): @@ -1690,10 +1743,15 @@ released A """ _a = IntMockBuffer("A", range(4 * 9), shape=(4, 9)) - cdef_nogil(_a) + assert cdef_nogil(_a) == 4 cdef int[:, :] a = _a print a[2, 7] + cdef int length + with nogil: + length = cdef_nogil(a) + assert length == 4 + @testcase def test_convert_slicenode_to_indexnode(): """ @@ -1706,7 +1764,7 @@ 2 released A """ - cdef int[:] a = IntMockBuffer("A", range(10), shape=(10,)) + cdef int[:] a = IntMockBuffer("A", range(10), shape=(10,)) # , writable=False) with nogil: a = a[2:4] print a[0] @@ -1716,10 +1774,10 @@ @cython.wraparound(False) def test_memslice_prange(arg): """ - >>> test_memslice_prange(IntMockBuffer("A", range(400), shape=(20, 4, 5))) + >>> test_memslice_prange(IntMockBuffer("A", range(400), shape=(20, 4, 5))) # FIXME: , writable=False)) acquired A released A - >>> test_memslice_prange(IntMockBuffer("A", range(200), shape=(100, 2, 1))) + >>> test_memslice_prange(IntMockBuffer("A", range(200), shape=(100, 2, 1))) # FIXME: , writable=False)) acquired A released A """ @@ -1853,11 +1911,7 @@ """ cdef TestAttrs[10] array cdef TestAttrs[:] struct_memview = array - - if sys.version_info[:2] >= (2, 7): - print builtins.memoryview(struct_memview).format - else: - print "T{i:int_attrib:c:char_attrib:}" + print builtins.memoryview(struct_memview).format # Test padding at the end of structs in the buffer support @@ -2145,7 +2199,7 @@ 7 8 9 - 2 5 + 5 1 5 """ cdef int i @@ -2166,10 +2220,12 @@ print m2[i] obj = m2[5] - print get_refcount(obj), obj + refcount1 = get_refcount(obj) + print obj del m2 - print get_refcount(obj), obj + refcount2 = get_refcount(obj) + print refcount1 - refcount2, obj assert unique_refcount == get_refcount(unique), (unique_refcount, get_refcount(unique)) @@ -2256,7 +2312,9 @@ """ >>> test_contig_scalar_to_slice_assignment() 14 14 14 14 + 30 14 30 14 20 20 20 20 + 30 30 20 20 """ cdef int[5][10] a cdef int[:, ::1] m = a @@ -2264,9 +2322,15 @@ m[...] = 14 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:, :1] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + m[:, :] = 20 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:1, :] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + @testcase def test_dtype_object_scalar_assignment(): """ @@ -2383,7 +2447,7 @@ @testcase def test_newaxis(int[:] one_D): """ - >>> A = IntMockBuffer("A", range(6)) + >>> A = IntMockBuffer("A", range(6)) # , writable=False) >>> test_newaxis(A) acquired A 3 @@ -2405,7 +2469,7 @@ @testcase def test_newaxis2(int[:, :] two_D): """ - >>> A = IntMockBuffer("A", range(6), shape=(3, 2)) + >>> A = IntMockBuffer("A", range(6), shape=(3, 2)) # , writable=False) >>> test_newaxis2(A) acquired A shape: 3 1 1 @@ -2439,3 +2503,61 @@ _print_attributes(d) +@testcase +def test_const_buffer(const int[:] a): + """ + >>> A = IntMockBuffer("A", range(6), shape=(6,), writable=False) + >>> test_const_buffer(A) + acquired A + 0 + 5 + released A + """ + cdef const int[:] c = a + print(a[0]) + print(c[-1]) + +@testcase +def test_loop(int[:] a, throw_exception): + """ + >>> A = IntMockBuffer("A", range(6), shape=(6,)) + >>> test_loop(A, False) + acquired A + 15 + released A + >>> try: + ... test_loop(A, True) + ... except ValueError: + ... pass + acquired A + released A + """ + cdef int sum = 0 + for ai in a: + sum += ai + if throw_exception: + raise ValueError() + print(sum) + +@testcase +def test_loop_reassign(int[:] a): + """ + >>> A = IntMockBuffer("A", range(6), shape=(6,)) + >>> test_loop_reassign(A) + acquired A + 0 + 1 + 2 + 3 + 4 + 5 + released A + 15 + """ + cdef int sum = 0 + for ai in a: + sum += ai + print(ai) + a = None # this should not mess up the loop though! + # release happens here, when the loop temp is released + print(sum) diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/numpy_memoryview.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/numpy_memoryview.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/numpy_memoryview.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/numpy_memoryview.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -12,11 +12,15 @@ cimport cython from cython cimport view -include "cythonarrayutil.pxi" +include "../testsupport/cythonarrayutil.pxi" include "../buffers/mockbuffers.pxi" ctypedef np.int32_t dtype_t +IS_PYPY = hasattr(sys, 'pypy_version_info') +NUMPY_VERSION = tuple(int(v) for v in np.__version__.split('.')[:2]) +print(NUMPY_VERSION) + def get_array(): # We need to type our array to get a __pyx_get_buffer() that typechecks # for np.ndarray and calls __getbuffer__ in numpy.pxd @@ -32,23 +36,21 @@ if x != args[0]: raise AssertionError(args) -__test__ = {} - -def testcase(f): - __test__[f.__name__] = f.__doc__ +def testcase_no_pypy(f, _is_pypy=hasattr(sys, "pypy_version_info")): + if _is_pypy: + f.__doc__ = "" # disable the tests return f -def testcase_numpy_1_5(f): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 5): - __test__[f.__name__] = f.__doc__ - return f +def gc_collect_if_required(): + if NUMPY_VERSION >= (1, 14) or IS_PYPY: + import gc + gc.collect() + # ### Test slicing memoryview slices # -@testcase def test_partial_slicing(array): """ >>> test_partial_slicing(a) @@ -64,7 +66,6 @@ ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis(array): """ >>> test_ellipsis(a) @@ -106,7 +107,6 @@ # ### Test slicing memoryview objects # -@testcase def test_partial_slicing_memoryview(array): """ >>> test_partial_slicing_memoryview(a) @@ -123,7 +123,6 @@ ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis_memoryview(array): """ >>> test_ellipsis_memoryview(a) @@ -164,7 +163,6 @@ ae(e.strides[0], e_obj.strides[0]) -@testcase def test_transpose(): """ >>> test_transpose() @@ -178,12 +176,12 @@ numpy_obj = np.arange(4 * 3, dtype=np.int32).reshape(4, 3) a = numpy_obj - a_obj = a + cdef object a_obj = a cdef dtype_t[:, :] b = a.T print a.T.shape[0], a.T.shape[1] print a_obj.T.shape - print numpy_obj.T.shape + print tuple(map(int, numpy_obj.T.shape)) # might use longs in Py2 cdef dtype_t[:, :] c with nogil: @@ -195,7 +193,6 @@ print a[3, 2], a.T[2, 3], a_obj[3, 2], a_obj.T[2, 3], numpy_obj[3, 2], numpy_obj.T[2, 3] -@testcase def test_transpose_type(a): """ >>> a = np.zeros((5, 10), dtype=np.float64) @@ -208,12 +205,8 @@ print m_transpose[6, 4] -@testcase_numpy_1_5 def test_numpy_like_attributes(cyarray): """ - For some reason this fails in numpy 1.4, with shape () and strides (40, 8) - instead of 20, 4 on my machine. Investigate this. - >>> cyarray = create_array(shape=(8, 5), mode="c") >>> test_numpy_like_attributes(cyarray) >>> test_numpy_like_attributes(cyarray.memview) @@ -229,14 +222,13 @@ cdef int[:, :] mslice = numarray assert ( mslice).base is numarray -@testcase_numpy_1_5 def test_copy_and_contig_attributes(a): """ >>> a = np.arange(20, dtype=np.int32).reshape(5, 4) >>> test_copy_and_contig_attributes(a) """ cdef np.int32_t[:, :] mslice = a - m = mslice + cdef object m = mslice # object copy # Test object copy attributes assert np.all(a == np.array(m.copy())) @@ -266,7 +258,7 @@ def index(array array): print build_numarray(array)[3, 2] -@testcase_numpy_1_5 +@testcase_no_pypy def test_coerce_to_numpy(): """ Test coercion to NumPy arrays, especially with automatically @@ -293,13 +285,13 @@ deallocating... 12.2 deallocating... - 13.3 + 13.25 deallocating... (14.4+15.5j) deallocating... - (16.6+17.7j) + (16.5+17.7j) deallocating... - (18.8+19.9j) + (18.8125+19.9375j) deallocating... 22 deallocating... @@ -347,6 +339,7 @@ 'e': 800, } + smallstructs[idx] = { 'a': 600, 'b': 700 } nestedstructs[idx] = { @@ -362,14 +355,15 @@ ints[idx] = 222 longlongs[idx] = 333 externs[idx] = 444 + assert externs[idx] == 444 # avoid "set but not used" C compiler warning floats[idx] = 11.1 doubles[idx] = 12.2 - longdoubles[idx] = 13.3 + longdoubles[idx] = 13.25 floatcomplex[idx] = 14.4 + 15.5j - doublecomplex[idx] = 16.6 + 17.7j - longdoublecomplex[idx] = 18.8 + 19.9j + doublecomplex[idx] = 16.5 + 17.7j + longdoublecomplex[idx] = 18.8125 + 19.9375j # x/64 to avoid float format rounding issues h_shorts[idx] = 22 h_doubles[idx] = 33.33 @@ -403,16 +397,16 @@ index( h_ushorts) -@testcase_numpy_1_5 +@testcase_no_pypy def test_memslice_getbuffer(): """ - >>> test_memslice_getbuffer() + >>> test_memslice_getbuffer(); gc_collect_if_required() [[ 0 2 4] [10 12 14]] callback called """ cdef int[:, :] array = create_array((4, 5), mode="c", use_callback=True) - print np.asarray(array)[::2, ::2] + print(np.asarray(array)[::2, ::2]) cdef class DeallocateMe(object): def __dealloc__(self): @@ -442,7 +436,6 @@ int a[4] signed char b[5] -@testcase_numpy_1_5 def test_memslice_structarray(data, dtype): """ >>> def b(s): return s.encode('ascii') @@ -498,7 +491,6 @@ print myslice[i].a[j] print myslice[i].b.decode('ASCII') -@testcase_numpy_1_5 def test_structarray_errors(StructArray[:] a): """ >>> dtype = np.dtype([('a', '4i'), ('b', '5b')]) @@ -545,7 +537,6 @@ def stringtest(String[:] view): pass -@testcase_numpy_1_5 def test_string_invalid_dims(): """ >>> def b(s): return s.encode('ascii') @@ -566,7 +557,6 @@ float attrib2 StringStruct attrib3 -@testcase_numpy_1_5 def test_struct_attributes(): """ >>> test_struct_attributes() @@ -622,7 +612,6 @@ getbuffer(self, info) info.suboffsets = self._shape -@testcase def test_null_strides(Buffer buffer_obj): """ >>> test_null_strides(Buffer()) @@ -642,7 +631,6 @@ assert m2[i, j] == buffer_obj.m[i, j], (i, j, m2[i, j], buffer_obj.m[i, j]) assert m3[i, j] == buffer_obj.m[i, j] -@testcase def test_null_strides_error(buffer_obj): """ >>> test_null_strides_error(Buffer()) @@ -691,3 +679,58 @@ a = np.arange(12).reshape([3, 4]) cdef np.int_t[:,:] a_view = a cdef np.int_t[:,:] b = a_view[1:2,:].T + + +@cython.boundscheck(False) +@cython.wraparound(False) +def test_boundscheck_and_wraparound(double[:, :] x): + """ + >>> import numpy as np + >>> array = np.ones((2,2)) * 3.5 + >>> test_boundscheck_and_wraparound(array) + """ + # Make sure we don't generate C compiler warnings for unused code here. + cdef Py_ssize_t numrow = x.shape[0] + cdef Py_ssize_t i + for i in range(numrow): + x[i, 0] + x[i] + x[i, ...] + x[i, :] + + +ctypedef struct SameTypeAfterArraysStructSimple: + double a[16] + double b[16] + double c + +def same_type_after_arrays_simple(): + """ + >>> same_type_after_arrays_simple() + """ + + cdef SameTypeAfterArraysStructSimple element + arr = np.ones(2, np.asarray(&element).dtype) + cdef SameTypeAfterArraysStructSimple[:] memview = arr + + +ctypedef struct SameTypeAfterArraysStructComposite: + int a + float b[8] + float c + unsigned long d + int e[5] + int f + int g + double h[4] + int i + +def same_type_after_arrays_composite(): + """ + >>> same_type_after_arrays_composite() if sys.version_info[:2] >= (3, 5) else None + >>> same_type_after_arrays_composite() if sys.version_info[:2] == (2, 7) else None + """ + + cdef SameTypeAfterArraysStructComposite element + arr = np.ones(2, np.asarray(&element).dtype) + cdef SameTypeAfterArraysStructComposite[:] memview = arr diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/numpy_memoryview_readonly.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/numpy_memoryview_readonly.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/numpy_memoryview_readonly.pyx 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/numpy_memoryview_readonly.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,172 @@ +# mode: run +# tag: readonly, const, numpy +# ticket: 1772 + +import numpy as np +cimport cython + +def new_array(dtype='float', writeable=True): + array = np.arange(10, dtype=dtype) + array.setflags(write=writeable) + return array + +ARRAY = new_array() + + +cdef getmax(const double[:] x): + """Example code, should work with both ro and rw memoryviews""" + cdef double max_val = -float('inf') + for val in x: + if val > max_val: + max_val = val + return max_val + + +cdef update_array(double [:] x): + """Modifying a ro memoryview should raise an error""" + x[0] = 23. + + +cdef getconst(const double [:] x): + """Should accept ro memoryviews""" + return x[0] + + +def test_mmview_rw(x): + """ + >>> test_mmview_rw(ARRAY) + 9.0 + """ + return getmax(x) + + +def test_mmview_ro(x): + """ + >>> test_mmview_ro(new_array()) + 9.0 + """ + x.setflags(write=False) + assert x.flags.writeable is False + return getmax(x) + + +def test_update_mmview_rw(x): + """ + >>> test_update_mmview_rw(new_array()) + 23.0 + """ + update_array(x) + return x[0] + + +def test_update_mmview_ro(x): + """ + >>> test_update_mmview_ro(new_array()) + 0.0 + """ + x.setflags(write=False) + assert x.flags.writeable is False + try: + update_array(x) + except ValueError: pass + else: + assert False, "RO error not raised!" + return getconst(x) + + +def test_rw_call_getmax(double[:] x): + """ + >>> test_rw_call_getmax(new_array()) + 23.0 + """ + update_array(x) + assert getconst(x) == 23 + return getmax(x) + + +def test_const_mmview_ro(x): + """ + >>> test_const_mmview_ro(new_array()) + 0.0 + """ + x.setflags(write=False) + assert x.flags.writeable is False + return getconst(x) + + +def test_two_views(x): + """ + >>> test_two_views(new_array()) + 23.0 + """ + cdef double[:] rw = x + cdef const double[:] ro = rw + rw[0] = 23 + return ro[0] + + +def test_assign_ro_to_rw(x): + """ + >>> test_assign_ro_to_rw(new_array()) + 2.0 + """ + cdef const double[:] ro = x + cdef double[:] rw = np.empty_like(ro) + rw[:] = ro + return rw[2] + + +def test_copy(): + """ + >>> test_copy() + (1.0, 2.0, 1.0, 1.0, 1.0, 2.0) + """ + cdef const double[:] ro = np.ones(3) + cdef const double[:] ro2 = ro.copy() + cdef double[:] rw = ro.copy() + cdef double[:] rw2 = ro2.copy() + rw[1] = 2 + rw2[2] = 2 + return rw[0], rw[1], rw[2], rw2[0], rw2[1], rw2[2] + + +cdef getmax_floating(const cython.floating[:] x): + """Function with fused type, should work with both ro and rw memoryviews""" + cdef cython.floating max_val = - float('inf') + for val in x: + if val > max_val: + max_val = val + return max_val + + +def test_mmview_const_fused_cdef(): + """Test cdef function with const fused type memory view as argument. + + >>> test_mmview_const_fused_cdef() + """ + cdef float[:] data_rw = new_array(dtype='float32') + assert getmax_floating(data_rw) == 9 + + cdef const float[:] data_ro = new_array(dtype='float32', writeable=False) + assert getmax_floating(data_ro) == 9 + + +def test_mmview_const_fused_def(const cython.floating[:] x): + """Test def function with const fused type memory view as argument. + + With read-write numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=True)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=True)) + 0.0 + + With read-only numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=False)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=False)) + 0.0 + """ + cdef cython.floating result = x[0] + return result diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/relaxed_strides.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/relaxed_strides.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/relaxed_strides.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/relaxed_strides.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -10,12 +10,12 @@ See also: Mailing list threads: - http://thread.gmane.org/gmane.comp.python.cython.devel/14762 - http://thread.gmane.org/gmane.comp.python.cython.devel/14634 + https://thread.gmane.org/gmane.comp.python.cython.devel/14762 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634 Detailed discussion of the difference between numpy/cython's current definition of "contiguity", and the correct definition: - http://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 The PR implementing NPY_RELAXED_STRIDES_CHECKING: https://github.com/numpy/numpy/pull/3162 @@ -62,3 +62,25 @@ """ cdef double[::1] a = array return a + +def test_zero_sized_multidim_ccontig(array): + """ + >>> contig = np.ascontiguousarray(np.zeros((4,4,4))[::2, 2:2, ::2]) + >>> _ = test_zero_sized_multidim_ccontig(contig) + + >>> a = np.zeros((4,4,4))[::2, 2:2, ::2] + >>> if NUMPY_HAS_RELAXED_STRIDES: _ = test_zero_sized_multidim_ccontig(a) + """ + cdef double[:, :, ::1] a = array + return a + +def test_zero_sized_multidim_fcontig(array): + """ + >>> contig = np.ascontiguousarray(np.zeros((4,4,4))[::2, 2:2, ::2]) + >>> _ = test_zero_sized_multidim_fcontig(contig) + + >>> a = np.zeros((4,4,4))[::2, 2:2, ::2] + >>> if NUMPY_HAS_RELAXED_STRIDES: _ = test_zero_sized_multidim_fcontig(a) + """ + cdef double[::1, :, :] a = array + return a diff -Nru cython-0.20.1+1~201611251650-6686/tests/memoryview/view_return_errors.pyx cython-0.20.1+1~202203241016-9537/tests/memoryview/view_return_errors.pyx --- cython-0.20.1+1~201611251650-6686/tests/memoryview/view_return_errors.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/memoryview/view_return_errors.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -13,7 +13,18 @@ raise TypeError('dummy') -def propagate(i): +cdef double[:] foo_nogil(int i) nogil: + if i == 1: + raise AttributeError('dummy') + if i == 2: + raise RuntimeError('dummy') + if i == 3: + raise ValueError('dummy') + if i == 4: + raise TypeError('dummy') + + +def propagate(i, bint nogil=False): """ >>> propagate(0) TypeError('Memoryview return value is not initialized') @@ -25,9 +36,20 @@ ValueError('dummy') >>> propagate(4) TypeError('dummy') + + >>> propagate(0, nogil=True) + TypeError('Memoryview return value is not initialized') + >>> propagate(1, nogil=True) + AttributeError('dummy') + >>> propagate(2, nogil=True) + RuntimeError('dummy') + >>> propagate(3, nogil=True) + ValueError('dummy') + >>> propagate(4, nogil=True) + TypeError('dummy') """ try: - foo(i) + foo_nogil(i) if nogil else foo(i) except Exception as e: print '%s(%r)' % (e.__class__.__name__, e.args[0]) else: diff -Nru cython-0.20.1+1~201611251650-6686/tests/pypy2_bugs.txt cython-0.20.1+1~202203241016-9537/tests/pypy2_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/pypy2_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pypy2_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,34 @@ +# Specific bugs that only apply to pypy2 + +build.cythonize_script +build.cythonize_script_package +run.initial_file_path +run.reduce_pickle +run.final_in_pxd +run.cdef_multiple_inheritance +run.cdef_multiple_inheritance_nodict +run.extstarargs +run.cpython_capi +run.isnot +run.partial_circular_import + +# pypy 2 seems to be preferring .py files to .so files +# https://foss.heptapod.net/pypy/pypy/issues/3185 +run.language_level +run.pure_pxd + +# Silly error with doctest matching slightly different string outputs rather than +# an actual bug but one I can't easily resolve +run.with_gil + +# moved from "pypy_bugs" - work on Pypy3, segfault on pypy2 +broken_exception +yield_from_pep380 + +# looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3 +memoryview.numpy_memoryview + +# type features that are disabled in PyPy2: +# (the overridden __Pyx_PyObject_GetItem requires CYTHON_USE_TYPE_SLOTS) +run.test_genericclass +run.test_subclassinit diff -Nru cython-0.20.1+1~201611251650-6686/tests/pypy_bugs.txt cython-0.20.1+1~202203241016-9537/tests/pypy_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/pypy_bugs.txt 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pypy_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -2,44 +2,62 @@ # either in PyPy, PyPy's cpyext, or Cython under PyPy, # which will be skipped in the normal testing run. -broken_exception bufaccess -buffmt -exarkun -memoryview -memslice -sequential_parallel -setjmp -yield_from_pep380 -memoryview_inplace_division - -# GIL issues -# https://bitbucket.org/pypy/pypy/issues/1778/pygilstate_ensure-should-not-deadlock -run.exceptions_nogil -run.nogil -run.with_gil -run.parallel -run.pstats_profile_test +memoryview.memoryview$ + +# looks like a doctest issue? It's failing to match output that looks +# identical to my eye +#run.unicodemethods +# multiphase import not supported +run.unicode_imports + +run.tp_new +run.test_fstring +run.test_exceptions +run.test_dictviews +run.str_subclass_kwargs +run.special_method_docstrings +run.slice_ptr +compile.min_async +run.cython_includes +run.test_unicode +run.__getattribute__ +run.__getattribute_subclasses__ +run.__debug__ +run.builtin_abs +run.builtincomplex +run.cdef_multiple_inheritance_errors +run.cdivision_CEP_516 +run.cyfunction +run.final_cdef_class +run.index +run.pyclass_special_methods +run.reimport_from_package +run.reimport_from_subinterpreter +run.reimport_failure +pkg.cimportfrom +embedded +TestCyCache +run.ext_auto_richcmp +run.coverage_cmd +run.coverage_cmd_src_layout +run.coverage_installed_pkg +run.coverage_api +run.coverage_nogil + +# very little coroutine-related seems to work +run.test_asyncgen +run.test_coroutines_pep492 +run.async_iter_pep492 +run.embedsignatures +run.py35_asyncio_async_def +run.asyncio_generators +run.py35_pep492_interop # gc issue? -memoryview_in_subclasses external_ref_reassignment run.exttype_dealloc -# https://bitbucket.org/pypy/pypy/issue/2023/cpyext-pydict_keys-values-items-does-not -builtin_subtype_methods_cy3 - # bugs in cpyext run.special_methods_T561 run.special_methods_T561_py2 - -# tests for things that don't exist in cpyext -compile.pylong -run.datetime_pxd -run.datetime_cimport -run.datetime_members -run.extern_builtins_T258 -run.unicode_ascii_auto_encoding -run.unicode_default_auto_encoding -run.str_ascii_auto_encoding -run.str_default_auto_encoding diff -Nru cython-0.20.1+1~201611251650-6686/tests/pypy_crash_bugs.txt cython-0.20.1+1~202203241016-9537/tests/pypy_crash_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/pypy_crash_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pypy_crash_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,12 @@ +# Bugs that causes hard crashes that we certainly don't +# want to run because it will break the testsuite + +# segfault +run.fastcall +memslice + +# gc issue? +memoryview_in_subclasses + +# """Fatal RPython error: NotImplementedError""" +pep442_tp_finalize diff -Nru cython-0.20.1+1~201611251650-6686/tests/pypy_implementation_detail_bugs.txt cython-0.20.1+1~202203241016-9537/tests/pypy_implementation_detail_bugs.txt --- cython-0.20.1+1~201611251650-6686/tests/pypy_implementation_detail_bugs.txt 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pypy_implementation_detail_bugs.txt 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,48 @@ +# PyPy "bugs" that are probably implementation differences from +# CPython rather than actual bugs. Therefore they aren't targets +# to be fixed (but there *may* be other details in the testfile +# that should be tested on PyPy?) + +run.starargs + +# refcounting-specific tests +double_dealloc_T796 +run.exceptionrefcount +run.capiimpl +run.refcount_in_meth +sequential_parallel +# Ideally just disable the reference-counting tests on PyPy? +run.fused_types +run.generator_frame_cycle +run.generators_in_refcycles +run.generators_py +run.parallel + +# "sys.getsizeof(object, default) will always return default on PyPy, and +# raise a TypeError if default is not provided." +buildenv + +# tests for things that don't exist in cpyext +compile.pylong +run.datetime_pxd +run.datetime_cimport +run.datetime_members +run.extern_builtins_T258 +run.line_trace +run.line_profile_test +run.pstats_profile_test +run.longintrepr + +# tests probably rely on immediate GC (although maybe the tests could be tweaked so +# only these bits don't run in PyPy?) +buffers.buffer +buffers.userbuffer +memoryview.cythonarray +memoryview.memoryview_pep484_typing +run.cpp_classes +run.cpp_classes_def + +# relies on cimport array (which has a different structure in PyPy) +memoryview_inplace_division +run.pyarray +run.array_cimport diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_basic.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_basic.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_basic.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_basic.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -6,6 +6,8 @@ import os.path import pyximport +pyximport.DEBUG_IMPORT = True + pyximport.install(build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) def test(): diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_errors.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_errors.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_errors.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_errors.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -7,6 +7,8 @@ from contextlib import contextmanager import pyximport +pyximport.DEBUG_IMPORT = True + pyximport.install(build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) @contextmanager diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_namespace.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_namespace.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_namespace.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_namespace.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,20 @@ + +PYTHON -c "import basic_test; basic_test.test()" + +######## basic_test.py ######## + +import os.path +import pyximport + +pyximport.DEBUG_IMPORT = True + +pyximport.install(build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) + +def test(): + import nsmod.mymodule + assert nsmod.mymodule.test_string == "TEST" + assert not nsmod.mymodule.__file__.rstrip('oc').endswith('.py'), nsmod.mymodule.__file__ + +######## nsmod/mymodule.pyx ######## + +test_string = "TEST" diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyimport_only.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyimport_only.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyimport_only.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyimport_only.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,25 @@ + +PYTHON -c "import pyimport_test; pyimport_test.test()" + +######## pyimport_test.py ######## + +import os.path +import pyximport + +pyximport.DEBUG_IMPORT = True + +pyximport.install(pyximport=False, pyimport=True, + build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) + +def test(): + import mymodule + assert mymodule.test_string == "TEST" + assert mymodule.compiled + +######## mymodule.py ######## + +import cython + +compiled = cython.compiled + +test_string = "TEST" diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyimport.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyimport.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyimport.srctree 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyimport.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -8,7 +8,8 @@ # blacklist for speed import pyximport.pyxbuild, Cython.Compiler.Pipeline -import distutils.core, distutils.ccompiler, distutils.command.build + +pyximport.DEBUG_IMPORT = True pyximport.install(pyximport=False, pyimport=True, build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) @@ -16,8 +17,12 @@ def test(): import mymodule assert mymodule.test_string == "TEST" - assert not mymodule.__file__.rstrip('oc').endswith('.py'), mymodule.__file__ + assert mymodule.compiled ######## mymodule.py ######## +import cython + +compiled = cython.compiled + test_string = "TEST" diff -Nru cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyxbld.srctree cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyxbld.srctree --- cython-0.20.1+1~201611251650-6686/tests/pyximport/pyximport_pyxbld.srctree 1970-01-01 00:00:00.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/pyximport/pyximport_pyxbld.srctree 2022-03-24 10:16:46.000000000 +0000 @@ -0,0 +1,37 @@ + +PYTHON -c "import basic_test; basic_test.test()" + +######## basic_test.py ######## + +import os.path +import pyximport + +pyximport.DEBUG_IMPORT = True + +pyximport.install(build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) + +def test(): + import mymodule + assert mymodule.test_string == "TEST" + assert mymodule.header_value == 5 + assert not mymodule.__file__.rstrip('oc').endswith('.py'), mymodule.__file__ + +######## mymodule.pyxbld ######## + +def make_ext(modname, pyxfilename): + from distutils.extension import Extension + return Extension(name = modname, + sources=[pyxfilename], + include_dirs=['./headers'] ) + +######## mymodule.pyx ######## + +cdef extern from "myheader.h": + int test_value + +header_value = test_value +test_string = "TEST" + +######## headers/myheader.h ######## + +static int test_value = 5; diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/addop.pyx cython-0.20.1+1~202203241016-9537/tests/run/addop.pyx --- cython-0.20.1+1~201611251650-6686/tests/run/addop.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/run/addop.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -166,3 +166,20 @@ ... except TypeError: pass """ return 2**30 + x + + +def add0(x): + """ + >>> add0(0) + (0, 0) + >>> add0(1) + (1, 1) + >>> add0(-1) + (-1, -1) + >>> a, b = add0(2**32) + >>> bigint(a) + 4294967296 + >>> bigint(b) + 4294967296 + """ + return x + 0, 0 + x diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/always_allow_keywords_T295.pyx cython-0.20.1+1~202203241016-9537/tests/run/always_allow_keywords_T295.pyx --- cython-0.20.1+1~201611251650-6686/tests/run/always_allow_keywords_T295.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/run/always_allow_keywords_T295.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,30 +1,44 @@ -# ticket: 295 +# mode: run +# ticket: t295 cimport cython +import sys +IS_PY2 = sys.version_info[0] == 2 + + +def assert_typeerror_no_keywords(func, *args, **kwds): + # Python 3.9 produces an slightly different error message + # to previous versions, so doctest isn't matching the + # traceback + try: + func(*args, **kwds) + except TypeError as e: + assert e.args[0].endswith(" takes no keyword arguments"), e.args[0] + else: + assert False, "call did not raise TypeError" + + +def func0(): + """ + >>> func0() + >>> func0(**{}) + """ def func1(arg): """ >>> func1(None) >>> func1(*[None]) >>> func1(arg=None) - Traceback (most recent call last): - ... - TypeError: func1() takes no keyword arguments """ - pass @cython.always_allow_keywords(False) def func2(arg): """ >>> func2(None) >>> func2(*[None]) - >>> func2(arg=None) - Traceback (most recent call last): - ... - TypeError: func2() takes no keyword arguments + >>> assert_typeerror_no_keywords(func2, arg=None) """ - pass @cython.always_allow_keywords(True) def func3(arg): @@ -35,32 +49,132 @@ """ pass + cdef class A: """ - >>> A().meth1(None) - >>> A().meth1(*[None]) - >>> A().meth1(arg=None) - Traceback (most recent call last): - ... - TypeError: meth1() takes no keyword arguments - >>> A().meth2(None) - >>> A().meth2(*[None]) - >>> A().meth2(arg=None) - Traceback (most recent call last): - ... - TypeError: meth2() takes no keyword arguments - >>> A().meth3(None) - >>> A().meth3(*[None]) - >>> A().meth3(arg=None) + >>> class PyA(object): + ... def meth0(self): pass + ... def meth1(self, arg): pass + + >>> PyA().meth0() + >>> PyA.meth0(PyA()) + >>> if not IS_PY2: PyA.meth0(self=PyA()) + >>> try: PyA().meth0(self=PyA()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + + >>> PyA().meth1(1) + >>> PyA.meth1(PyA(), 1) + >>> PyA.meth1(PyA(), arg=1) + >>> if not IS_PY2: PyA.meth1(self=PyA(), arg=1) """ + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> A().meth0_nokw() + >>> A().meth0_nokw(**{}) + >>> try: pass #A.meth0_nokw(self=A()) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> A().meth0_kw() + >>> A().meth0_kw(**{}) + >>> A.meth0_kw(A()) + >>> #A.meth0_kw(self=A()) + >>> try: pass #A().meth0_kw(self=A()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) + def meth1_kw(self, arg): + """ + >>> A().meth1_kw(None) + >>> A().meth1_kw(*[None]) + >>> A().meth1_kw(arg=None) + >>> A.meth1_kw(A(), arg=None) + >>> #A.meth1_kw(self=A(), arg=None) + """ + + @cython.always_allow_keywords(False) + def meth1_nokw(self, arg): + """ + >>> A().meth1_nokw(None) + >>> A().meth1_nokw(*[None]) + >>> assert_typeerror_no_keywords(A().meth1_nokw, arg=None) + >>> assert_typeerror_no_keywords(A.meth1_nokw, A(), arg=None) + >>> try: pass # A.meth1_nokw(self=A(), arg=None) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass # assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(False) + def meth2(self, arg): + """ + >>> A().meth2(None) + >>> A().meth2(*[None]) + >>> assert_typeerror_no_keywords(A().meth2, arg=None) + """ + + @cython.always_allow_keywords(True) + def meth3(self, arg): + """ + >>> A().meth3(None) + >>> A().meth3(*[None]) + >>> A().meth3(arg=None) + """ + + +class B(object): + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> B().meth0_nokw() + >>> B().meth0_nokw(**{}) + >>> if not IS_PY2: assert_typeerror_no_keywords(B.meth0_nokw, self=B()) + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> B().meth0_kw() + >>> B().meth0_kw(**{}) + >>> B.meth0_kw(B()) + >>> if not IS_PY2: B.meth0_kw(self=B()) + >>> try: B().meth0_kw(self=B()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) def meth1(self, arg): - pass + """ + >>> B().meth1(None) + >>> B().meth1(*[None]) + >>> B().meth1(arg=None) + >>> B.meth1(B(), arg=None) + >>> if not IS_PY2: B.meth1(self=B(), arg=None) + """ @cython.always_allow_keywords(False) def meth2(self, arg): - pass + """ + >>> B().meth2(None) + >>> B().meth2(*[None]) + >>> B.meth2(B(), None) + >>> if not IS_PY2: B.meth2(self=B(), arg=None) + >>> B().meth2(arg=None) # assert_typeerror_no_keywords(B().meth2, arg=None) -> not a cdef class! + """ @cython.always_allow_keywords(True) def meth3(self, arg): - pass + """ + >>> B().meth3(None) + >>> B().meth3(*[None]) + >>> B().meth3(arg=None) + """ diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/annotate_html.pyx cython-0.20.1+1~202203241016-9537/tests/run/annotate_html.pyx --- cython-0.20.1+1~201611251650-6686/tests/run/annotate_html.pyx 2016-11-25 16:50:54.000000000 +0000 +++ cython-0.20.1+1~202203241016-9537/tests/run/annotate_html.pyx 2022-03-24 10:16:46.000000000 +0000 @@ -1,15 +1,18 @@ """ +>>> from codecs import open >>> import os.path as os_path >>> module_path = os_path.join(os_path.dirname(__file__), os_path.basename(__file__).split('.', 1)[0]) >>> assert module_path.endswith('annotate_html') >>> assert os_path.exists(module_path + '.c') or os_path.exists(module_path + '.cpp'), module_path >>> assert os_path.exists(module_path + '.html'), module_path ->>> with open(module_path + '.html') as html_file: +>>> with open(module_path + '.html', 'r', 'utf8') as html_file: ... html = html_file.read() >>> import re >>> assert re.search('
    ', html)
+>>> from Cython.Compiler.Annotate import AnnotationCCodeWriter
+>>> assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code
 """
 
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/annotation_typing.pyx cython-0.20.1+1~202203241016-9537/tests/run/annotation_typing.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/annotation_typing.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/annotation_typing.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,19 +1,30 @@
-# cython: annotation_typing=True
+# mode: run
+# tag: pep484, warnings
 
+cimport cython
 from cython cimport typeof
+from cpython.ref cimport PyObject
 
+try:
+    from typing import Optional
+except ImportError:
+    pass
 
-def pytypes_def(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4) -> list:
+
+def old_dict_syntax(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4) -> list:
     """
-    >>> pytypes_def([1])
+    >>> old_dict_syntax([1])
     ('list object', 'int', 'long', 'float')
     [1, 2, 3, 4.0]
-    >>> pytypes_def([1], 3)
+    >>> old_dict_syntax([1], 3)
     ('list object', 'int', 'long', 'float')
     [1, 3, 3, 4.0]
-    >>> pytypes_def(123)
+    >>> old_dict_syntax(123)
     Traceback (most recent call last):
     TypeError: Argument 'a' has incorrect type (expected list, got int)
+    >>> old_dict_syntax(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got NoneType)
     """
     print(typeof(a), typeof(b), typeof(c), typeof(d))
     a.append(b)
@@ -22,41 +33,77 @@
     return a
 
 
-cpdef pytypes_cpdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4):
+def pytypes_def(a: list, b: int = 2, c: long = 3, d: float = 4, n: list = None, o: Optional[tuple] = ()) -> list:
+    """
+    >>> pytypes_def([1])
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 2, 3, 4.0, None, ()]
+    >>> pytypes_def([1], 3)
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 3, 3, 4.0, None, ()]
+    >>> pytypes_def([1], 3, 2, 1, [], None)
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 3, 2, 1.0, [], None]
+    >>> pytypes_def(123)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got int)
+    >>> pytypes_def(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got NoneType)
+    """
+    print(typeof(a), typeof(b), typeof(c), typeof(d), typeof(n), typeof(o))
+    a.append(b)
+    a.append(c)
+    a.append(d)
+    a.append(n)
+    a.append(o)
+    return a
+
+
+cpdef pytypes_cpdef(a: list, b: int = 2, c: long = 3, d: float = 4, n: list = None, o: Optional[tuple] = ()):
     """
     >>> pytypes_cpdef([1])
-    ('list object', 'int', 'long', 'float')
-    [1, 2, 3, 4.0]
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 2, 3, 4.0, None, ()]
     >>> pytypes_cpdef([1], 3)
-    ('list object', 'int', 'long', 'float')
-    [1, 3, 3, 4.0]
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 3, 3, 4.0, None, ()]
+    >>> pytypes_cpdef([1], 3, 2, 1, [], None)
+    ('list object', 'Python object', 'Python object', 'double', 'list object', 'tuple object')
+    [1, 3, 2, 1.0, [], None]
     >>> pytypes_cpdef(123)
     Traceback (most recent call last):
     TypeError: Argument 'a' has incorrect type (expected list, got int)
+    >>> pytypes_cpdef(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got NoneType)
     """
-    print(typeof(a), typeof(b), typeof(c), typeof(d))
+    print(typeof(a), typeof(b), typeof(c), typeof(d), typeof(n), typeof(o))
     a.append(b)
     a.append(c)
     a.append(d)
+    a.append(n)
+    a.append(o)
     return a
 
 
-cdef c_pytypes_cdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4):
-    print(typeof(a), typeof(b), typeof(c), typeof(d))
+cdef c_pytypes_cdef(a: list, b: int = 2, c: long = 3, d: float = 4, n: list = None):
+    print(typeof(a), typeof(b), typeof(c), typeof(d), typeof(n))
     a.append(b)
     a.append(c)
     a.append(d)
+    a.append(n)
     return a
 
 
 def pytypes_cdef(a, b=2, c=3, d=4):
     """
     >>> pytypes_cdef([1])
-    ('list object', 'int', 'long', 'float')
-    [1, 2, 3, 4.0]
+    ('list object', 'Python object', 'Python object', 'double', 'list object')
+    [1, 2, 3, 4.0, None]
     >>> pytypes_cdef([1], 3)
-    ('list object', 'int', 'long', 'float')
-    [1, 3, 3, 4.0]
+    ('list object', 'Python object', 'Python object', 'double', 'list object')
+    [1, 3, 3, 4.0, None]
     >>> pytypes_cdef(123)   # doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: ...
@@ -64,6 +111,25 @@
     return c_pytypes_cdef(a, b, c, d)
 
 
+def ctypes_def(a: list, b: cython.int = 2, c: cython.long = 3, d: cython.float = 4) -> list:
+    """
+    >>> ctypes_def([1])
+    ('list object', 'int', 'long', 'float')
+    [1, 2, 3, 4.0]
+    >>> ctypes_def([1], 3)
+    ('list object', 'int', 'long', 'float')
+    [1, 3, 3, 4.0]
+    >>> ctypes_def(123)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got int)
+    """
+    print(typeof(a), typeof(b), typeof(c), typeof(d))
+    a.append(b)
+    a.append(c)
+    a.append(d)
+    return a
+
+
 def return_tuple_for_carray() -> tuple:
     """
     >>> return_tuple_for_carray()
@@ -72,3 +138,210 @@
     cdef int[3] x
     x = [1, 2, 3]
     return x
+
+
+MyStruct = cython.struct(x=cython.int, y=cython.int, data=cython.double)
+
+
+@cython.ccall
+def struct_io(s : MyStruct) -> MyStruct:
+    """
+    >>> d = struct_io(dict(x=1, y=2, data=3))
+    >>> sorted(d.items())
+    [('data', 3.0), ('x', 2), ('y', 1)]
+    >>> d = struct_io(None)
+    Traceback (most recent call last):
+    TypeError: Expected a mapping, got NoneType
+    """
+    t = s
+    t.x, t.y = s.y, s.x
+    return t
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceFromPyTypeNode",
+    "//SimpleCallNode//CoerceToPyTypeNode",
+)
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//SimpleCallNode",
+)
+def call_struct_io(s : MyStruct) -> MyStruct:
+    """
+    >>> d = call_struct_io(dict(x=1, y=2, data=3))
+    >>> sorted(d.items())
+    [('data', 3.0), ('x', 2), ('y', 1)]
+    >>> d = call_struct_io(None)
+    Traceback (most recent call last):
+    TypeError: Expected a mapping, got NoneType
+    """
+    return struct_io(s)
+
+
+@cython.test_assert_path_exists(
+    "//CFuncDefNode",
+    "//CFuncDefNode//DefNode",
+    "//CFuncDefNode[@return_type]",
+    "//CFuncDefNode[@return_type.is_struct_or_union = True]",
+)
+@cython.ccall
+def struct_convert(d) -> MyStruct:
+    """
+    >>> d = struct_convert(dict(x=1, y=2, data=3))
+    >>> sorted(d.items())
+    [('data', 3.0), ('x', 1), ('y', 2)]
+    >>> struct_convert({})  # make sure we can raise exceptions through struct return values
+    Traceback (most recent call last):
+    ValueError: No value specified for struct attribute 'x'
+    """
+    return d
+
+
+@cython.test_assert_path_exists(
+    "//CFuncDefNode",
+    "//CFuncDefNode//DefNode",
+    "//CFuncDefNode[@return_type]",
+    "//CFuncDefNode[@return_type.is_int = True]",
+)
+@cython.ccall
+def exception_default(raise_exc : cython.bint = False) -> cython.int:
+    """
+    >>> exception_default(raise_exc=False)
+    10
+    >>> exception_default(raise_exc=True)
+    Traceback (most recent call last):
+    ValueError: huhu!
+    """
+    if raise_exc:
+        raise ValueError("huhu!")
+    return 10
+
+
+def call_exception_default(raise_exc=False):
+    """
+    >>> call_exception_default(raise_exc=False)
+    10
+    >>> call_exception_default(raise_exc=True)
+    Traceback (most recent call last):
+    ValueError: huhu!
+    """
+    return exception_default(raise_exc)
+
+
+@cython.test_assert_path_exists(
+    "//CFuncDefNode",
+    "//CFuncDefNode//DefNode",
+    "//CFuncDefNode[@return_type]",
+    "//CFuncDefNode[@return_type.is_int = True]",
+)
+@cython.ccall
+def exception_default_uint(raise_exc : cython.bint = False) -> cython.uint:
+    """
+    >>> print(exception_default_uint(raise_exc=False))
+    10
+    >>> exception_default_uint(raise_exc=True)
+    Traceback (most recent call last):
+    ValueError: huhu!
+    """
+    if raise_exc:
+        raise ValueError("huhu!")
+    return 10
+
+
+def call_exception_default_uint(raise_exc=False):
+    """
+    >>> print(call_exception_default_uint(raise_exc=False))
+    10
+    >>> call_exception_default_uint(raise_exc=True)
+    Traceback (most recent call last):
+    ValueError: huhu!
+    """
+    return exception_default_uint(raise_exc)
+
+
+class EarlyClass(object):
+    """
+    >>> a = EarlyClass(1)
+    >>> a.string_forward_declaration()  # should probably raise an error at some point
+    1
+    >>> x = LateClass()
+    >>> a = EarlyClass(x)
+    >>> x2 = a.string_forward_declaration()
+    >>> assert x is x2, x2
+    """
+    def __init__(self, x):
+        self.x = x
+    def string_forward_declaration(self) -> 'LateClass':
+        return self.x
+
+class LateClass(object):
+    pass
+
+
+def py_float_default(price : float=None, ndigits=4):
+    """
+    Python default arguments should prevent C type inference.
+
+    >>> py_float_default()
+    (None, 4)
+    >>> py_float_default(2)
+    (2, 4)
+    >>> py_float_default(2.0)
+    (2.0, 4)
+    >>> py_float_default(2, 3)
+    (2, 3)
+    """
+    return price, ndigits
+
+
+cdef class ClassAttribute:
+    cls_attr : float = 1.
+
+
+@cython.cfunc
+def take_ptr(obj: cython.pointer(PyObject)):
+    pass
+
+def call_take_ptr():
+    """
+    >>> call_take_ptr()  # really just a compile-test
+    """
+    python_dict = {"abc": 123}
+    take_ptr(cython.cast(cython.pointer(PyObject), python_dict))
+
+@cython.cclass
+class HasPtr:
+    """
+    >>> HasPtr()
+    HasPtr(1, 1)
+    """
+    a: cython.pointer(cython.int)
+    b: cython.int
+
+    def __init__(self):
+        self.b = 1
+        self.a = cython.address(self.b)
+    def __repr__(self):
+        return f"HasPtr({self.a[0]}, {self.b})"
+
+
+_WARNINGS = """
+14:32: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+14:47: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.
+14:56: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+14:77: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.
+14:85: Python type declaration in signature annotation does not refer to a Python type
+14:85: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+36:64: PEP-484 recommends 'typing.Optional[...]' for arguments that can be None.
+63:68: PEP-484 recommends 'typing.Optional[...]' for arguments that can be None.
+90:68: PEP-484 recommends 'typing.Optional[...]' for arguments that can be None.
+274:44: Unknown type declaration in annotation, ignoring
+281:29: Ambiguous types in annotation, ignoring
+298:15: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute?
+# BUG:
+63:6: 'pytypes_cpdef' redeclared
+146:0: 'struct_io' redeclared
+181:0: 'struct_convert' redeclared
+200:0: 'exception_default' redeclared
+231:0: 'exception_default_uint' redeclared
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/args_unpacking_in_closure_T658.pyx cython-0.20.1+1~202203241016-9537/tests/run/args_unpacking_in_closure_T658.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/args_unpacking_in_closure_T658.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/args_unpacking_in_closure_T658.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 658
+# ticket: t658
 
 def outer(int x, *args, **kwargs):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/argument_unpacking_closure_T736.py cython-0.20.1+1~202203241016-9537/tests/run/argument_unpacking_closure_T736.py
--- cython-0.20.1+1~201611251650-6686/tests/run/argument_unpacking_closure_T736.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/argument_unpacking_closure_T736.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 736
+# ticket: t736
 # tag: default arguments, closure
 
 def default_args_for_closure(a=1, b=2):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/arithmetic_analyse_types_helper.h cython-0.20.1+1~202203241016-9537/tests/run/arithmetic_analyse_types_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/arithmetic_analyse_types_helper.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/arithmetic_analyse_types_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-/* A set of mutually incompatable return types. */
+/* A set of mutually incompatible return types. */
 
 struct short_return { char *msg; };
 struct int_return { char *msg; };
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/arithmetic_analyse_types.pyx cython-0.20.1+1~202203241016-9537/tests/run/arithmetic_analyse_types.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/arithmetic_analyse_types.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/arithmetic_analyse_types.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 676
+# ticket: t676
 # tag: cpp
 
 from cython cimport typeof
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/array_cimport.srctree cython-0.20.1+1~202203241016-9537/tests/run/array_cimport.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/array_cimport.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/array_cimport.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -32,6 +32,6 @@
 
 cdef array a = array('i', [1,2,3])
 cdef Foo x
-print a.data.as_ints[0]
+print(a.data.as_ints[0])
 x = Foo(a)
-print x.obj.data.as_ints[0]
+print(x.obj.data.as_ints[0])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ass2global.py cython-0.20.1+1~202203241016-9537/tests/run/ass2global.py
--- cython-0.20.1+1~201611251650-6686/tests/run/ass2global.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ass2global.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,9 +1,20 @@
+# mode: run
+# tag: pyglobal
+
 """
-    >>> getg()
-    5
-    >>> setg(42)
-    >>> getg()
-    42
+>>> getg()
+5
+>>> getg()
+5
+>>> getg()
+5
+>>> setg(42)
+>>> getg()
+42
+>>> getg()
+42
+>>> getg()
+42
 """
 
 g = 5
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/assert.pyx cython-0.20.1+1~202203241016-9537/tests/run/assert.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/assert.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/assert.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,6 +2,10 @@
 
 cimport cython
 
+@cython.test_assert_path_exists(
+    '//AssertStatNode',
+    '//AssertStatNode//RaiseStatNode',
+)
 def f(a, b, int i):
     """
     >>> f(1, 2, 1)
@@ -22,7 +26,9 @@
 
 @cython.test_assert_path_exists(
     '//AssertStatNode',
-    '//AssertStatNode//TupleNode')
+    '//AssertStatNode//RaiseStatNode',
+    '//AssertStatNode//RaiseStatNode//TupleNode',
+)
 def g(a, b):
     """
     >>> g(1, "works")
@@ -38,7 +44,9 @@
 
 @cython.test_assert_path_exists(
     '//AssertStatNode',
-    '//AssertStatNode//TupleNode')
+    '//AssertStatNode//RaiseStatNode',
+    '//AssertStatNode//RaiseStatNode//TupleNode',
+)
 def g(a, b):
     """
     >>> g(1, "works")
@@ -54,8 +62,9 @@
 
 @cython.test_assert_path_exists(
     '//AssertStatNode',
-    '//AssertStatNode//TupleNode',
-    '//AssertStatNode//TupleNode//TupleNode')
+    '//AssertStatNode//RaiseStatNode',
+    '//AssertStatNode//RaiseStatNode//TupleNode',
+    '//AssertStatNode//RaiseStatNode//TupleNode//TupleNode',)
 def assert_with_tuple_arg(a):
     """
     >>> assert_with_tuple_arg(True)
@@ -67,9 +76,12 @@
 
 
 @cython.test_assert_path_exists(
-    '//AssertStatNode')
+    '//AssertStatNode',
+    '//AssertStatNode//RaiseStatNode',
+)
 @cython.test_fail_if_path_exists(
-    '//AssertStatNode//TupleNode')
+    '//AssertStatNode//TupleNode',
+)
 def assert_with_str_arg(a):
     """
     >>> assert_with_str_arg(True)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/assigned_builtin_methods.pyx cython-0.20.1+1~202203241016-9537/tests/run/assigned_builtin_methods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/assigned_builtin_methods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/assigned_builtin_methods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -30,7 +30,7 @@
 @cython.test_fail_if_path_exists(
     '//ReturnStatNode//PythonCapiCallNode')
 @cython.test_assert_path_exists(
-    '//ReturnStatNode//SimpleCallNode')
+    '//ReturnStatNode//PyMethodCallNode')
 def bound_dict_get_reassign(dict d):
     """
     >>> bound_dict_get_reassign({})
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/async_def.pyx cython-0.20.1+1~202203241016-9537/tests/run/async_def.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/async_def.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/async_def.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,62 @@
+# cython: language_level=3str, binding=True
+# mode: run
+# tag: pep492, await, gh3337
+
+"""
+Cython specific tests in addition to "test_coroutines_pep492.pyx"
+(which is copied from CPython).
+"""
+
+import sys
+
+
+def run_async(coro):
+    #assert coro.__class__ is types.GeneratorType
+    assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__
+
+    buffer = []
+    result = None
+    while True:
+        try:
+            buffer.append(coro.send(None))
+        except StopIteration as ex:
+            result = ex.value if sys.version_info >= (3, 5) else ex.args[0] if ex.args else None
+            break
+    return buffer, result
+
+
+async def test_async_temp_gh3337(x, y):
+    """
+    >>> run_async(test_async_temp_gh3337(2, 3))
+    ([], -1)
+    >>> run_async(test_async_temp_gh3337(3, 2))
+    ([], 0)
+    """
+    return min(x - y, 0)
+
+
+async def outer_with_nested(called):
+    """
+    >>> called = []
+    >>> _, inner = run_async(outer_with_nested(called))
+    >>> called  # after outer_with_nested()
+    ['outer', 'make inner', 'deco', 'return inner']
+    >>> _ = run_async(inner())
+    >>> called  # after inner()
+    ['outer', 'make inner', 'deco', 'return inner', 'inner']
+    """
+    called.append('outer')
+
+    def deco(f):
+        called.append('deco')
+        return f
+
+    called.append('make inner')
+
+    @deco
+    async def inner():
+        called.append('inner')
+        return 1
+
+    called.append('return inner')
+    return inner
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/async_globals.pyx cython-0.20.1+1~202203241016-9537/tests/run/async_globals.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/async_globals.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/async_globals.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,20 @@
+# mode: run
+# tag: pep492, asyncfor, await, gh2613
+
+# Using C-globals in coroutines.
+
+
+cdef object py_retval
+
+
+async def test():
+    """
+    >>> t = test()
+    >>> try: t.send(None)
+    ... except StopIteration as ex:
+    ...     print(ex.args[0] if ex.args else None)
+    ... else: print("NOT STOPPED!")
+    None
+    """
+    global py_retval
+    py_retval = {'foo': 42}
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/asyncio_generators.srctree cython-0.20.1+1~202203241016-9537/tests/run/asyncio_generators.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/asyncio_generators.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/asyncio_generators.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -30,7 +30,9 @@
     result = loop.run_until_complete(task())
     assert 3 == result, result
 
-runloop(from_asyncio_import.wait3)
+import sys
+if sys.version_info < (3, 7):
+    runloop(from_asyncio_import.wait3)
 
 
 ######## test_import.py ########
@@ -43,7 +45,9 @@
     result = loop.run_until_complete(task())
     assert 3 == result, result
 
-runloop(import_asyncio.wait3)
+import sys
+if sys.version_info < (3, 7):
+    runloop(import_asyncio.wait3)
 
 
 ######## test_async_def.py ########
@@ -89,6 +93,7 @@
 import asyncio
 
 ASYNCIO_SUPPORTS_COROUTINE = sys.version_info[:2] >= (3, 5)
+ASYNCIO_SUPPORTS_YIELD_FROM = sys.version_info[:2] < (3, 7)
 
 def runloop(task):
     loop = asyncio.get_event_loop()
@@ -96,23 +101,28 @@
     assert 3 == result, result
 
 import import_asyncio
-runloop(import_asyncio.wait3)       # 1a)
+if ASYNCIO_SUPPORTS_YIELD_FROM:
+    runloop(import_asyncio.wait3)       # 1a)
 import from_asyncio_import
-runloop(from_asyncio_import.wait3)  # 1b)
+if ASYNCIO_SUPPORTS_YIELD_FROM:
+    runloop(from_asyncio_import.wait3)  # 1b)
 
 import async_def
 if ASYNCIO_SUPPORTS_COROUTINE:
-    runloop(async_def.wait3)        # 1c)
+    runloop(async_def.wait3)            # 1c)
 
-runloop(from_asyncio_import.wait3)  # 2a)
-runloop(import_asyncio.wait3)       # 2b)
+if ASYNCIO_SUPPORTS_YIELD_FROM:
+    runloop(from_asyncio_import.wait3)  # 2a)
+    runloop(import_asyncio.wait3)       # 2b)
 if ASYNCIO_SUPPORTS_COROUTINE:
-    runloop(async_def.wait3)        # 2c)
+    runloop(async_def.wait3)            # 2c)
 
-runloop(from_asyncio_import.wait3)  # 3a)
-runloop(import_asyncio.wait3)       # 3b)
+import sys
+if ASYNCIO_SUPPORTS_YIELD_FROM:
+    runloop(from_asyncio_import.wait3)  # 3a)
+    runloop(import_asyncio.wait3)       # 3b)
 if ASYNCIO_SUPPORTS_COROUTINE:
-    runloop(async_def.wait3)        # 3c)
+    runloop(async_def.wait3)            # 3c)
 
 try:
     from collections.abc import Generator
@@ -160,7 +170,6 @@
 
 import asyncio
 
-@asyncio.coroutine
 @types_coroutine
 def wait3():
     counter = 0
@@ -179,9 +188,8 @@
 except ImportError:
     types_coroutine = lambda f:f
 
-from asyncio import coroutine, sleep
+from asyncio import sleep
 
-@coroutine
 @types_coroutine
 def wait3():
     counter = 0
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/async_iter_pep492.pyx cython-0.20.1+1~202203241016-9537/tests/run/async_iter_pep492.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/async_iter_pep492.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/async_iter_pep492.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -175,7 +175,7 @@
     >>> testfunc = test_broken_anext()
     >>> try: run_async(testfunc())
     ... except TypeError as exc:
-    ...     assert ' int ' in str(exc)
+    ...     assert ' int' in str(exc)
     ... else:
     ...     print("NOT RAISED!")
     """
@@ -279,9 +279,10 @@
     print(I[0])
 
 
-cdef class AI_old:
-    async def __aiter__(self):
-        1/0
+# old-style pre-3.5.2 AIter protocol - no longer supported
+#cdef class AI_old:
+#    async def __aiter__(self):
+#        1/0
 
 
 cdef class AI_new:
@@ -291,9 +292,9 @@
 
 def test_aiter_raises(AI):
     """
-    >>> test_aiter_raises(AI_old)
-    RAISED
-    0
+    #>>> test_aiter_raises(AI_old)
+    #RAISED
+    #0
     >>> test_aiter_raises(AI_new)
     RAISED
     0
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bad_c_struct_T252.pyx cython-0.20.1+1~202203241016-9537/tests/run/bad_c_struct_T252.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bad_c_struct_T252.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bad_c_struct_T252.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 252
+# ticket: t252
 
 cdef cf(default=None):
     return default
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/binop_reverse_methods_GH2056.pyx cython-0.20.1+1~202203241016-9537/tests/run/binop_reverse_methods_GH2056.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/binop_reverse_methods_GH2056.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/binop_reverse_methods_GH2056.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,265 @@
+cimport cython
+
+import sys
+IS_PYTHON2 = sys.version_info[0] == 2
+
+__doc__ = ""
+
+
+@cython.c_api_binop_methods(False)
+@cython.cclass
+class Base(object):
+    """
+    >>> Base() + 2
+    'Base.__add__(Base(), 2)'
+    >>> 2 + Base()
+    'Base.__radd__(Base(), 2)'
+
+    >>> Base(implemented=False) + 2  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    >>> 2 + Base(implemented=False)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+
+    >>> Base() ** 2
+    'Base.__pow__(Base(), 2, None)'
+    >>> 2 ** Base()
+    'Base.__rpow__(Base(), 2, None)'
+    >>> pow(Base(), 2, 100)
+    'Base.__pow__(Base(), 2, 100)'
+    """
+    implemented: cython.bint
+
+    def __init__(self, *, implemented=True):
+        self.implemented = implemented
+
+    def __add__(self, other):
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
+            return "Base.__add__(%s, %s)" % (self, other)
+        else:
+            return NotImplemented
+
+    def __radd__(self, other):
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
+            return "Base.__radd__(%s, %s)" % (self, other)
+        else:
+            return NotImplemented
+
+    def __pow__(self, other, mod):
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
+            return "Base.__pow__(%s, %s, %s)" % (self, other, mod)
+        else:
+            return NotImplemented
+
+    def __rpow__(self, other, mod):
+        assert cython.typeof(self) == "Base"
+        if self.implemented:
+            return "Base.__rpow__(%s, %s, %s)" % (self, other, mod)
+        else:
+            return NotImplemented
+
+    def __repr__(self):
+        return "%s()" % (self.__class__.__name__)
+
+
+@cython.c_api_binop_methods(False)
+@cython.cclass
+class OverloadLeft(Base):
+    """
+    >>> OverloadLeft() + 2
+    'OverloadLeft.__add__(OverloadLeft(), 2)'
+    >>> 2 + OverloadLeft()
+    'Base.__radd__(OverloadLeft(), 2)'
+
+    >>> OverloadLeft() + Base()
+    'OverloadLeft.__add__(OverloadLeft(), Base())'
+    >>> Base() + OverloadLeft()
+    'Base.__add__(Base(), OverloadLeft())'
+
+    >>> OverloadLeft(implemented=False) + Base(implemented=False)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    >>> Base(implemented=False) + OverloadLeft(implemented=False)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    """
+    derived_implemented: cython.bint
+
+    def __init__(self, *, implemented=True):
+        super().__init__(implemented=implemented)
+        self.derived_implemented = implemented
+
+    def __add__(self, other):
+        assert cython.typeof(self) == "OverloadLeft"
+        if self.derived_implemented:
+            return "OverloadLeft.__add__(%s, %s)" % (self, other)
+        else:
+            return NotImplemented
+
+
+@cython.c_api_binop_methods(False)
+@cython.cclass
+class OverloadRight(Base):
+    """
+    >>> OverloadRight() + 2
+    'Base.__add__(OverloadRight(), 2)'
+    >>> 2 + OverloadRight()
+    'OverloadRight.__radd__(OverloadRight(), 2)'
+
+    >>> OverloadRight() + Base()
+    'Base.__add__(OverloadRight(), Base())'
+    >>> Base() + OverloadRight()
+    'OverloadRight.__radd__(OverloadRight(), Base())'
+
+    >>> OverloadRight(implemented=False) + Base(implemented=False)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    >>> Base(implemented=False) + OverloadRight(implemented=False)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    """
+    derived_implemented: cython.bint
+
+    def __init__(self, *, implemented=True):
+        super().__init__(implemented=implemented)
+        self.derived_implemented = implemented
+
+    def __radd__(self, other):
+        assert cython.typeof(self) == "OverloadRight"
+        if self.derived_implemented:
+            return "OverloadRight.__radd__(%s, %s)" % (self, other)
+        else:
+            return NotImplemented
+
+
+@cython.c_api_binop_methods(True)
+@cython.cclass
+class OverloadCApi(Base):
+    """
+    >>> OverloadCApi() + 2
+    'OverloadCApi.__add__(OverloadCApi(), 2)'
+    >>> 2 + OverloadCApi()
+    'OverloadCApi.__add__(2, OverloadCApi())'
+
+    >>> OverloadCApi() + Base()
+    'OverloadCApi.__add__(OverloadCApi(), Base())'
+    >>> Base() + OverloadCApi()
+    'OverloadCApi.__add__(Base(), OverloadCApi())'
+
+    >>> OverloadCApi(derived_implemented=False) + 2 #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    >>> 2 + OverloadCApi(derived_implemented=False) #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: unsupported operand type...
+    """
+    derived_implemented: cython.bint
+
+    def __init__(self, *, derived_implemented=True):
+        super().__init__(implemented=True)
+        self.derived_implemented = derived_implemented
+
+    def __add__(self, other):
+        assert cython.typeof(self) != "OverloadCApi"  # should be untyped
+        if isinstance(self, OverloadCApi):
+            derived_implemented = (self).derived_implemented
+        else:
+            derived_implemented = (other).derived_implemented
+        if derived_implemented:
+            return "OverloadCApi.__add__(%s, %s)" % (self, other)
+        else:
+            return NotImplemented
+
+
+if sys.version_info >= (3, 5):
+    __doc__ += """
+    >>> d = PyVersionDependent()
+    >>> d @ 2
+    9
+    >>> 2 @ d
+    99
+    >>> i = d
+    >>> i @= 2
+    >>> i
+    999
+"""
+
+
+@cython.c_api_binop_methods(False)
+@cython.cclass
+class PyVersionDependent:
+    """
+    >>> d = PyVersionDependent()
+    >>> d / 2
+    5
+    >>> 2 / d
+    2
+    >>> d // 2
+    55
+    >>> 2 // d
+    22
+    >>> i = d
+    >>> i /= 2
+    >>> i
+    4
+    >>> i = d
+    >>> i //= 2
+    >>> i
+    44
+    """
+    def __div__(self, other):
+        assert IS_PYTHON2
+        return 5
+
+    def __rdiv__(self, other):
+        assert IS_PYTHON2
+        return 2
+
+    def __idiv__(self, other):
+        assert IS_PYTHON2
+        return 4
+
+    def __truediv__(self, other):
+        assert not IS_PYTHON2
+        return 5
+
+    def __rtruediv__(self, other):
+        assert not IS_PYTHON2
+        return 2
+
+    def __itruediv__(self, other):
+        assert not IS_PYTHON2
+        return 4
+
+    def __floordiv__(self, other):
+        return 55
+
+    def __rfloordiv__(self, other):
+        return 22
+
+    def __ifloordiv__(self, other):
+        return 44
+
+    def __matmul__(self, other):
+        return 9
+
+    def __rmatmul__(self, other):
+        return 99
+
+    def __imatmul__(self, other):
+        return 999
+
+
+# TODO: Test a class that only defines the `__r...__()` methods.
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bint_binop_T145.pyx cython-0.20.1+1~202203241016-9537/tests/run/bint_binop_T145.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bint_binop_T145.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bint_binop_T145.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 145
+# ticket: t145
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bint_property_T354.pyx cython-0.20.1+1~202203241016-9537/tests/run/bint_property_T354.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bint_property_T354.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bint_property_T354.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 354
+# ticket: t354
 
 cdef class Test:
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bound_builtin_methods_T589.pyx cython-0.20.1+1~202203241016-9537/tests/run/bound_builtin_methods_T589.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bound_builtin_methods_T589.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bound_builtin_methods_T589.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 589
+# ticket: t589
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_abs.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_abs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_abs.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_abs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,6 @@
 # mode: run
-# ticket: 698
+# ticket: t698
+# distutils: extra_compile_args=-fwrapv
 
 cdef extern from *:
     int INT_MAX
@@ -7,7 +8,7 @@
 
 max_int = INT_MAX
 max_long = LONG_MAX
-max_long_long = 2 ** (sizeof(long long) * 8 - 1) - 1
+max_long_long = (2) ** (sizeof(long long) * 8 - 1) - 1
 
 
 cimport cython
@@ -31,25 +32,58 @@
     return abs(a)
 
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
-                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']")
+                                "//ReturnStatNode//NameNode[@entry.cname = 'abs']")
+def sub_abs(int a):
+    """
+    >>> sub_abs(5)
+    (-5, 95)
+    >>> sub_abs(105)
+    (-105, -5)
+    """
+    return -abs(a), 100 - abs(a)
+
+@cython.overflowcheck(True)
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'abs']")
 def int_abs(int a):
     """
     >>> int_abs(-5) == 5
     True
     >>> int_abs(-5.1) == 5
     True
-    >>> int_abs(-max_int-1) > 0
-    True
-    >>> int_abs(-max_int-1) == abs(-max_int-1)   or (max_int, int_abs(-max_int-1), abs(-max_int-1))
-    True
+    >>> int_abs(-max_int-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> int_abs(max_int) == abs(max_int)         or (max_int, int_abs(max_int), abs(max_int))
     True
     """
     return abs(a)
 
+@cython.overflowcheck(True)
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'abs']")
+cdef int c_int_abs(int a) nogil except *:
+    return abs(a)
+
+def test_c_int_abs(int a):
+    """
+    >>> test_c_int_abs(-5) == 5
+    True
+    >>> test_c_int_abs(-5.1) == 5
+    True
+    >>> test_c_int_abs(-max_int-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
+    >>> test_c_int_abs(max_int) == abs(max_int)  or (max_int, test_c_int_abs(max_int), abs(max_int))
+    True
+    """
+    return c_int_abs(a)
+
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
-@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']",
-                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def uint_abs(unsigned int a):
     """
     >>> uint_abs(max_int) == abs(max_int)         or (max_int, uint_abs(max_int), abs(max_int))
@@ -57,48 +91,120 @@
     """
     return abs(a)
 
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
+cdef unsigned int c_uint_abs(unsigned int a) nogil:
+    return abs(a)
+
+def test_c_uint_abs(unsigned int a):
+    """
+    >>> test_c_uint_abs(max_int) == abs(max_int)  or (max_int, test_c_uint_abs(max_int), abs(max_int))
+    True
+    """
+    return c_uint_abs(a)
+
+@cython.overflowcheck(True)
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
-                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+                                "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def long_abs(long a):
     """
     >>> long_abs(-5) == 5
     True
     >>> long_abs(-5.1) == 5
     True
-    >>> long_abs(-max_long-1) > 0
-    True
-    >>> long_abs(-max_long-1) == abs(-max_long-1)   or (max_long, long_abs(-max_long-1), abs(-max_long-1))
-    True
+    >>> long_abs(-max_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> long_abs(max_long) == abs(max_long)         or (max_long, long_abs(max_long), abs(max_long))
     True
     """
     return abs(a)
 
+@cython.overflowcheck(True)
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
+cdef long c_long_abs(long a) nogil except *:
+    return abs(a)
+
+def test_c_long_abs(long a):
+    """
+    >>> test_c_long_abs(-5) == 5
+    True
+    >>> test_c_long_abs(-5.1) == 5
+    True
+    >>> test_c_long_abs(-max_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
+    >>> test_c_long_abs(max_long) == abs(max_long)  or (max_long, test_c_long_abs(max_long), abs(max_long))
+    True
+    """
+    return c_long_abs(a)
+
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
-@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_int']",
-                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_long']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
 def ulong_abs(unsigned long a):
     """
     >>> ulong_abs(max_long) == abs(max_long)         or (max_int, ulong_abs(max_long), abs(max_long))
     True
+    >>> ulong_abs(max_long + 5) == abs(max_long + 5)         or (max_long + 5, ulong_abs(max_long + 5), abs(max_long + 5))
+    True
     """
     return abs(a)
 
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']")
+@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']",
+                                 "//ReturnStatNode//NameNode[@entry.cname = 'labs']")
+cdef unsigned long c_ulong_abs(unsigned long a) nogil:
+    return abs(a)
+
+def test_c_ulong_abs(unsigned long a):
+    """
+    >>> test_c_ulong_abs(max_long) == abs(max_long)  or (max_int, test_c_ulong_abs(max_long), abs(max_long))
+    True
+    >>> test_c_ulong_abs(max_long + 5) == abs(max_long + 5)  or (max_long + 5, test_c_ulong_abs(max_long + 5), abs(max_long + 5))
+    True
+    """
+    return c_ulong_abs(a)
+
+@cython.overflowcheck(True)
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
                                 "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']")
 def long_long_abs(long long a):
     """
     >>> long_long_abs(-(2**33)) == 2**33
     True
-    >>> long_long_abs(-max_long_long-1) > 0
-    True
-    >>> long_long_abs(-max_long_long-1) == abs(-max_long_long-1)  or (max_long_long, long_long_abs(-max_long_long-1), abs(-max_long_long-1))
-    True
+    >>> long_long_abs(-max_long_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
     >>> long_long_abs(max_long_long) == abs(max_long_long)        or (max_long_long, long_long_abs(max_long_long), abs(max_long_long))
     True
     """
     return abs(a)
 
+@cython.overflowcheck(True)
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']")
+cdef long long c_long_long_abs(long long a) nogil except *:
+    return abs(a)
+
+def test_c_long_long_abs(long long a):
+    """
+    >>> test_c_long_long_abs(-(2**33)) == 2**33
+    True
+    >>> test_c_long_long_abs(-max_long_long-1)     #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    OverflowError: ...
+    >>> test_c_long_long_abs(max_long_long) == abs(max_long_long) or (max_long_long, test_c_long_long_abs(max_long_long), abs(max_long_long))
+    True
+    """
+    return c_long_long_abs(a)
+
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
                                 "//ReturnStatNode//NameNode[@entry.cname = 'fabs']")
 def double_abs(double a):
@@ -111,6 +217,20 @@
     return abs(a)
 
 @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'fabs']")
+cdef double c_double_abs(double a) nogil:
+    return abs(a)
+
+def test_c_double_abs(double a):
+    """
+    >>> test_c_double_abs(-5)
+    5.0
+    >>> test_c_double_abs(-5.5)
+    5.5
+    """
+    return c_double_abs(a)
+
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
                                 "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']")
 def float_abs(float a):
     """
@@ -120,3 +240,42 @@
     5.5
     """
     return abs(a)
+
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']")
+cdef float c_float_abs(float a) nogil:
+    return abs(a)
+
+def test_c_float_abs(float a):
+    """
+    >>> test_c_float_abs(-5)
+    5.0
+    >>> test_c_float_abs(-5.5)
+    5.5
+    """
+    return c_float_abs(a)
+
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']")
+def complex_abs(complex a):
+    """
+    >>> complex_abs(-5j)
+    5.0
+    >>> complex_abs(-5.5j)
+    5.5
+    """
+    return abs(a)
+
+@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']",
+                                "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']")
+cdef double c_complex_abs(complex a) nogil:
+    return abs(a)
+
+def test_c_complex_abs(complex a):
+    """
+    >>> test_c_complex_abs(-5j)
+    5.0
+    >>> test_c_complex_abs(-5.5j)
+    5.5
+    """
+    return c_complex_abs(a)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_float.py cython-0.20.1+1~202203241016-9537/tests/run/builtin_float.py
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_float.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_float.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,20 @@
+# mode: run
+# tag: pure3.0
 
+import cython
 import sys
 
+def fix_underscores(s):
+    if sys.version_info < (3, 6) or getattr(sys, 'pypy_version_info', (9, 9)) < (3, 7, 4):
+        # Py2 float() does not support PEP-515 underscore literals
+        if isinstance(s, bytes):
+            if not cython.compiled and b'_' in s:
+                return s.replace(b'_', b'')
+        elif '_' in s:
+            return s.replace('_', '')
+    return s
+
+
 def empty_float():
     """
     >>> float()
@@ -11,24 +25,254 @@
     x = float()
     return x
 
+
 def float_conjugate():
     """
     >>> float_call_conjugate()
     1.5
     """
-    if sys.version_info >= (2,6):
-        x = 1.5 .conjugate()
-    else:
-        x = 1.5
+    x = 1.5 .conjugate()
     return x
 
+
 def float_call_conjugate():
     """
     >>> float_call_conjugate()
     1.5
     """
-    if sys.version_info >= (2,6):
-        x = float(1.5).conjugate()
-    else:
-        x = 1.5
+    x = float(1.5).conjugate()
     return x
+
+
+def from_int(i):
+    """
+    >>> from_int(0)
+    0.0
+    >>> from_int(1)
+    1.0
+    >>> from_int(-1)
+    -1.0
+    >>> from_int(99)
+    99.0
+    >>> from_int(-99)
+    -99.0
+
+    >>> for exp in (14, 15, 16, 30, 31, 32, 52, 53, 54, 60, 61, 62, 63, 64):
+    ...     for sign in (1, 0, -1):
+    ...         value = (sign or 1) * 2**exp + sign
+    ...         float_value = from_int(value)
+    ...         assert float_value == float(value), "expected %s2**%s+%s == %r, got %r, difference %r" % (
+    ...             '-' if sign < 0 else '', exp, sign, float(value), float_value, float_value - float(value))
+    """
+    return float(i)
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_bytes(s: bytes):
+    """
+    >>> from_bytes(b"123")
+    123.0
+    >>> from_bytes(b"123.25")
+    123.25
+    >>> from_bytes(fix_underscores(b"98_5_6.2_1"))
+    9856.21
+    >>> from_bytes(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
+    >>> from_bytes(b"123E100")
+    1.23e+102
+    >>> from_bytes(b"12__._3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12__._3...
+    >>> from_bytes(b"_12.3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ..._12.3...
+    >>> from_bytes(b"12.3_")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12.3_...
+    >>> from_bytes(b"na_n")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...na_n...
+    >>> from_bytes(None)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError...
+    """
+    return float(s)
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_bytes_literals():
+    """
+    >>> from_bytes_literals()
+    (123.0, 123.23, 123.76, 1e+100)
+    """
+    return float(b"123"), float(b"123.23"), float(fix_underscores(b"12_3.7_6")), float(b"1e100")
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_bytearray(s: bytearray):
+    """
+    >>> from_bytearray(bytearray(b"123"))
+    123.0
+    >>> from_bytearray(bytearray(b"123.25"))
+    123.25
+    >>> from_bytearray(bytearray(fix_underscores(b"98_5_6.2_1")))
+    9856.21
+    >>> from_bytearray(bytearray(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")))
+    1.2413112312318938e+47
+    >>> from_bytearray(bytearray(b"123E100"))
+    1.23e+102
+    >>> from_bytearray(bytearray(b"12__._3"))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12__._3...
+    >>> from_bytearray(bytearray(b"_12.3"))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ..._12.3...
+    >>> from_bytearray(bytearray(b"12.3_"))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12.3_...
+    >>> from_bytearray(bytearray(b"in_f"))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...in_f...
+    >>> from_bytearray(None)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError...
+    """
+    return float(s)
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_str(s: 'str'):
+    """
+    >>> from_str("123")
+    123.0
+    >>> from_str("123.25")
+    123.25
+    >>> from_str(fix_underscores("3_21.2_5"))
+    321.25
+    >>> from_str(fix_underscores("12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
+    >>> from_str("123E100")
+    1.23e+102
+    >>> from_str("12__._3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12__._3...
+    >>> from_str("_12.3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ..._12.3...
+    >>> from_str("12.3_")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12.3_...
+    >>> from_str("n_an")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...n_an...
+    >>> from_str(None)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError...
+    """
+    return float(s)
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_str_literals():
+    """
+    >>> from_str_literals()
+    (123.0, 123.23, 124.23, 1e+100)
+    """
+    return float("123"), float("123.23"), float(fix_underscores("1_2_4.2_3")), float("1e100")
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_unicode(s: 'unicode'):
+    """
+    >>> from_unicode(u"123")
+    123.0
+    >>> from_unicode(u"123.25")
+    123.25
+    >>> from_unicode(fix_underscores(u"12_4.8_5"))
+    124.85
+    >>> from_unicode(fix_underscores(u"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))
+    1.2413112312318938e+47
+    >>> from_unicode(u"123E100")
+    1.23e+102
+    >>> from_unicode(u"123.23\\N{PUNCTUATION SPACE}")
+    123.23
+    >>> from_unicode(u"\\N{PUNCTUATION SPACE} 123.23 \\N{PUNCTUATION SPACE}")
+    123.23
+    >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} 12_3.2_3 \\N{PUNCTUATION SPACE}"))
+    123.23
+    >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " + u"\\N{PUNCTUATION SPACE} " * 22)  # >= 40 chars
+    123.54
+    >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} " * 25 + u"1_23.5_4 " + u"\\N{PUNCTUATION SPACE} " * 22))
+    123.54
+    >>> from_unicode(u"\\N{PUNCTUATION SPACE} " + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE}")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...123.54 123.54...
+    >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE} " * 22)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...123.54 123.54...
+    >>> from_unicode(u"_12__._3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ..._12__._3...
+    >>> from_unicode(u"_12.3")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ..._12.3...
+    >>> from_unicode(u"12.3_")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...12.3_...
+    >>> from_unicode(u"i_nf")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError: ...i_nf...
+    >>> from_unicode(None)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError...
+    """
+    return float(s)
+
+
+@cython.test_assert_path_exists(
+    "//CoerceToPyTypeNode",
+    "//CoerceToPyTypeNode//PythonCapiCallNode",
+)
+def from_unicode_literals():
+    """
+    >>> from_unicode_literals()
+    (123.0, 123.23, 123.45, 1e+100, 123.23)
+    """
+    return float(u"123"), float(u"123.23"), float(fix_underscores(u"12_3.4_5")), float(u"1e100"), float(u"123.23\N{PUNCTUATION SPACE}")
+
+
+def catch_valueerror(val):
+    """
+    >>> catch_valueerror("foo")
+    False
+    >>> catch_valueerror(u"foo")
+    False
+    >>> catch_valueerror(b"foo")
+    False
+    >>> catch_valueerror(bytearray(b"foo"))
+    False
+    >>> catch_valueerror("-1")
+    -1.0
+    """
+    try:
+        return float(val)
+    except ValueError:
+        return False
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_methods_return_values.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_methods_return_values.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_methods_return_values.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_methods_return_values.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: list, set, builtins
-# ticket: 688
+# ticket: t688
 
 _set = set
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_sorted.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_sorted.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_sorted.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_sorted.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,16 +1,20 @@
 cimport cython
 
+
 def generator():
     yield 2
     yield 1
     yield 3
 
+
 def returns_set():
     return {"foo", "bar", "baz"}
 
+
 def returns_tuple():
     return (1, 2, 3, 0)
 
+
 @cython.test_fail_if_path_exists("//SimpleCallNode")
 def sorted_arg(x):
     """
@@ -31,6 +35,26 @@
     """
     return sorted(x)
 
+
+@cython.test_assert_path_exists("//GeneralCallNode")
+def sorted_arg_with_key(x):
+    """
+    >>> a = [3, 2, 1]
+    >>> sorted_arg_with_key(a)
+    [3, 2, 1]
+    >>> a
+    [3, 2, 1]
+    >>> sorted_arg_with_key(generator())
+    [3, 2, 1]
+    >>> sorted_arg_with_key(returns_tuple())
+    [3, 2, 1, 0]
+    >>> sorted_arg_with_key(object())
+    Traceback (most recent call last):
+    TypeError: 'object' object is not iterable
+    """
+    return sorted(x, key=lambda x: -x)
+
+
 @cython.test_fail_if_path_exists("//YieldExprNode",
                                  "//NoneCheckNode")
 @cython.test_assert_path_exists("//InlinedGeneratorExpressionNode")
@@ -41,6 +65,7 @@
     """
     return sorted(i*i for i in range(10,0,-1))
 
+
 @cython.test_fail_if_path_exists("//SimpleCallNode//SimpleCallNode")
 @cython.test_assert_path_exists("//SimpleCallNode/NameNode[@name = 'range']")
 def sorted_list_of_range():
@@ -50,6 +75,7 @@
     """
     return sorted(list(range(10,0,-1)))
 
+
 @cython.test_fail_if_path_exists("//SimpleCallNode")
 def sorted_list_literal():
     """
@@ -58,6 +84,7 @@
     """
     return sorted([3, 1, 2] * 2)
 
+
 @cython.test_fail_if_path_exists("//SimpleCallNode")
 def sorted_tuple_literal():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_subtype_methods_cy3.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_subtype_methods_cy3.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_subtype_methods_cy3.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_subtype_methods_cy3.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # cython: language_level=3
 # mode: run
-# ticket: 653
+# ticket: t653
 
 
 class DictPySubtype(dict):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_subtype_methods_T653.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_subtype_methods_T653.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_subtype_methods_T653.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_subtype_methods_T653.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 #cython: language_level=2
 # mode: run
-# ticket: 653
+# ticket: t653
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_type_inheritance_T608.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_type_inheritance_T608.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_type_inheritance_T608.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_type_inheritance_T608.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 608
+# ticket: t608
 
 cdef class MyInt(int):
     """
@@ -125,3 +125,26 @@
     cdef readonly int value
     def __cinit__(self, value):
         self.value = value
+
+def test_exception_isinstance(maybe_exn):
+    """
+    >>> test_exception_isinstance(Exception())
+    True
+    >>> test_exception_isinstance(MyException(3))
+    True
+    >>> test_exception_isinstance(3)
+    False
+    """
+    return isinstance(maybe_exn, Exception)
+
+def test_exception_type_cast(Exception maybe_exn):
+    """
+    >>> test_exception_type_cast(Exception())
+    >>> test_exception_type_cast(MyException(3))
+    >>> test_exception_type_cast(3)   # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: Argument 'maybe_exn' has incorrect type (expected ...Exception, got int)
+    """
+    cdef object o = maybe_exn
+    cdef Exception e = o
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_types_class.py cython-0.20.1+1~202203241016-9537/tests/run/builtin_types_class.py
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_types_class.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_types_class.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,60 @@
+# mode: run
+
+from __future__ import print_function
+
+import cython
+
+# https://github.com/cython/cython/issues/3954
+# calls to the __class__ attributes of builtin types were optimized to something invalid
+
+@cython.locals(d=dict)
+def test_dict(d):
+    """
+    >>> test_dict({})
+    dict
+    {}
+    """
+    print(d.__class__.__name__)
+    print(d.__class__())
+
+@cython.locals(i=int)
+def test_int(i):
+    """
+    >>> test_int(0)
+    int
+    0
+    """
+    print(i.__class__.__name__)
+    print(i.__class__())
+
+@cython.cclass
+class C:
+    def __str__(self):
+        return "I'm a C object"
+
+@cython.locals(c=C)
+def test_cdef_class(c):
+    """
+    # This wasn't actually broken but is worth testing anyway
+    >>> test_cdef_class(C())
+    C
+    I'm a C object
+    """
+    print(c.__class__.__name__)
+    print(c.__class__())
+
+@cython.locals(d=object)
+def test_object(o):
+    """
+    >>> test_object({})
+    dict
+    {}
+    >>> test_object(1)
+    int
+    0
+    >>> test_object(C())
+    C
+    I'm a C object
+    """
+    print(o.__class__.__name__)
+    print(o.__class__())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/builtin_types_none_T166.pyx cython-0.20.1+1~202203241016-9537/tests/run/builtin_types_none_T166.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/builtin_types_none_T166.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/builtin_types_none_T166.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 166
+# ticket: t166
 
 __doc__ = u"""
 >>> l = None
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bytearray_coercion.pyx cython-0.20.1+1~202203241016-9537/tests/run/bytearray_coercion.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bytearray_coercion.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bytearray_coercion.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -38,6 +38,27 @@
     """
     return b[:2]
 
+
+def infer_concatenation_types(bytearray b):
+    """
+    >>> b = bytearray(b'a\\xFEc')
+    >>> b2, c, d, e, tb, tc, td, te = infer_concatenation_types(b)
+    >>> tb, tc, td, te
+    ('bytearray object', 'bytearray object', 'bytearray object', 'bytearray object')
+    >>> b2, c, d, e
+    (bytearray(b'a\\xfec'), bytearray(b'a\\xfeca\\xfec'), bytearray(b'a\\xfeca\\xfec'), bytearray(b'a\\xfeca\\xfec'))
+    """
+    c = b[:]
+    c += b[:]
+
+    d = b[:]
+    d *= 2
+
+    e = b + b
+
+    return b, c, d, e, cython.typeof(b), cython.typeof(c), cython.typeof(d), cython.typeof(e)
+
+
 def infer_index_types(bytearray b):
     """
     >>> b = bytearray(b'a\\xFEc')
@@ -51,11 +72,12 @@
         e = b[1]
     return c, d, e, cython.typeof(c), cython.typeof(d), cython.typeof(e), cython.typeof(b[1])
 
+
 def infer_slice_types(bytearray b):
     """
     >>> b = bytearray(b'abc')
     >>> print(infer_slice_types(b))
-    (bytearray(b'bc'), bytearray(b'bc'), bytearray(b'bc'), 'Python object', 'Python object', 'Python object', 'bytearray object')
+    (bytearray(b'bc'), bytearray(b'bc'), bytearray(b'bc'), 'bytearray object', 'bytearray object', 'bytearray object', 'bytearray object')
     """
     c = b[1:]
     with cython.boundscheck(False):
@@ -64,6 +86,7 @@
         e = b[1:]
     return c, d, e, cython.typeof(c), cython.typeof(d), cython.typeof(e), cython.typeof(b[1:])
 
+
 def assign_to_index(bytearray b, value):
     """
     >>> b = bytearray(b'0abcdefg')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bytearray_iter.py cython-0.20.1+1~202203241016-9537/tests/run/bytearray_iter.py
--- cython-0.20.1+1~201611251650-6686/tests/run/bytearray_iter.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bytearray_iter.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,90 @@
+# mode: run
+# tag: pure3, pure2
+
+import cython
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+@cython.locals(x=bytearray)
+def basic_bytearray_iter(x):
+    """
+    >>> basic_bytearray_iter(bytearray(b"hello"))
+    h
+    e
+    l
+    l
+    o
+    """
+    for a in x:
+        print(chr(a))
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+@cython.locals(x=bytearray)
+def reversed_bytearray_iter(x):
+    """
+    >>> reversed_bytearray_iter(bytearray(b"hello"))
+    o
+    l
+    l
+    e
+    h
+    """
+    for a in reversed(x):
+        print(chr(a))
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+@cython.locals(x=bytearray)
+def modifying_bytearray_iter1(x):
+    """
+    >>> modifying_bytearray_iter1(bytearray(b"abcdef"))
+    a
+    b
+    c
+    3
+    """
+    count = 0
+    for a in x:
+        print(chr(a))
+        del x[-1]
+        count += 1
+    print(count)
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+@cython.locals(x=bytearray)
+def modifying_bytearray_iter2(x):
+    """
+    >>> modifying_bytearray_iter2(bytearray(b"abcdef"))
+    a
+    c
+    e
+    3
+    """
+    count = 0
+    for a in x:
+        print(chr(a))
+        del x[0]
+        count += 1
+    print(count)
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+@cython.locals(x=bytearray)
+def modifying_reversed_bytearray_iter(x):
+    """
+    NOTE - I'm not 100% sure how well-defined this behaviour is in Python.
+    However, for the moment Python and Cython seem to do the same thing.
+    Testing that it doesn't crash is probably more important than the exact output!
+    >>> modifying_reversed_bytearray_iter(bytearray(b"abcdef"))
+    f
+    f
+    f
+    f
+    f
+    f
+    """
+    for a in reversed(x):
+        print(chr(a))
+        del x[0]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bytearraymethods.pyx cython-0.20.1+1~202203241016-9537/tests/run/bytearraymethods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bytearraymethods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bytearraymethods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -275,3 +275,15 @@
     b.append(i)
     b.append(o)
     return b
+
+
+cdef class BytearraySubtype(bytearray):
+    """
+    >>> b = BytearraySubtype(b'abc')
+    >>> b._append(ord('x'))
+    >>> b.append(ord('y'))
+    >>> print(b.decode('ascii'))
+    abcxy
+    """
+    def _append(self, x):
+        self.append(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bytes_formatting.pyx cython-0.20.1+1~202203241016-9537/tests/run/bytes_formatting.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bytes_formatting.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bytes_formatting.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,34 @@
+# mode: run
+# tag: stringformat, bytesformat
+
+
+import sys
+IS_PY2 = sys.version_info[0] < 3
+
+
+if IS_PY2:
+    __doc__ = """
+>>> print(format_bytes_with_str(u'abc'))
+1 12170405abc6A
+"""
+
+
+def format_bytes():
+    """
+    >>> print(format_bytes())
+    1 121704056A
+    """
+    cdef bytes result = b'%d%3i%x%02X%02.0f%g%c' % (
+        1, 12, 23, 4, 5, 6, 65)
+    assert type(result) is bytes
+    return result.decode('ascii')
+
+
+def format_bytes_with_str(s):
+    """
+    >>> print(format_bytes_with_str(b'abc'))
+    1 12170405abc6A
+    """
+    result = b'%d%3i%x%02X%02.0f%s%g%c' % (
+        1, 12, 23, 4, 5, s, 6, 65)
+    return result if IS_PY2 else result.decode('ascii')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/bytesmethods.pyx cython-0.20.1+1~202203241016-9537/tests/run/bytesmethods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/bytesmethods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/bytesmethods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,13 @@
 cimport cython
 
+cdef extern from *:
+    cdef Py_ssize_t PY_SSIZE_T_MIN
+    cdef Py_ssize_t PY_SSIZE_T_MAX
+
+SSIZE_T_MAX = PY_SSIZE_T_MAX
+SSIZE_T_MIN = PY_SSIZE_T_MIN
+
+
 b_a = b'a'
 b_b = b'b'
 
@@ -114,6 +122,14 @@
     
     >>> print(bytes_decode(s, -300, -500))
     
+    >>> print(bytes_decode(s, SSIZE_T_MIN, SSIZE_T_MIN))
+    
+    >>> print(bytes_decode(s, SSIZE_T_MIN, SSIZE_T_MAX))
+    abaab
+    >>> print(bytes_decode(s, SSIZE_T_MAX, SSIZE_T_MIN))
+    
+    >>> print(bytes_decode(s, SSIZE_T_MAX, SSIZE_T_MAX))
+    
 
     >>> s[:'test']                       # doctest: +ELLIPSIS
     Traceback (most recent call last):
@@ -153,6 +169,47 @@
 
 
 @cython.test_assert_path_exists(
+    "//PythonCapiCallNode")
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode")
+def bytes_decode_utf16(bytes s):
+    """
+    >>> s = 'abc'.encode('UTF-16')
+    >>> print(bytes_decode_utf16(s))
+    abc
+    """
+    return s.decode('utf16')
+
+
+@cython.test_assert_path_exists(
+    "//PythonCapiCallNode")
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode")
+def bytes_decode_utf16_le(bytes s):
+    """
+    >>> s = 'abc'.encode('UTF-16LE')
+    >>> assert s != 'abc'.encode('UTF-16BE')
+    >>> print(bytes_decode_utf16_le(s))
+    abc
+    """
+    return s.decode('utf_16_le')
+
+
+@cython.test_assert_path_exists(
+    "//PythonCapiCallNode")
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode")
+def bytes_decode_utf16_be(bytes s):
+    """
+    >>> s = 'abc'.encode('UTF-16BE')
+    >>> assert s != 'abc'.encode('UTF-16LE')
+    >>> print(bytes_decode_utf16_be(s))
+    abc
+    """
+    return s.decode('utf_16_be')
+
+
+@cython.test_assert_path_exists(
     "//PythonCapiCallNode")
 @cython.test_fail_if_path_exists(
     "//SimpleCallNode")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/callargs.pyx cython-0.20.1+1~202203241016-9537/tests/run/callargs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/callargs.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/callargs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -168,15 +168,15 @@
     """
     >>> test_int_kwargs(e)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(f)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(g)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> test_int_kwargs(h)     # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     """
     f(a=1,b=2,c=3, **{10:20,30:40})
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/carray_coercion.pyx cython-0.20.1+1~202203241016-9537/tests/run/carray_coercion.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/carray_coercion.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/carray_coercion.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -5,6 +5,10 @@
 IS_32BIT_PY2 = not IS_PY3 and sys.maxint < 2**32
 
 
+from libc cimport stdint
+from libc.stdint cimport int16_t as my_int16_t
+
+
 def unlongify(v):
     # on 32bit Py2.x platforms, 'unsigned int' coerces to a Python long => fix doctest output here.
     s = repr(v)
@@ -64,6 +68,30 @@
     v[0] = 1
     v[1] = 2
     v[2] = 3
+    return v
+
+
+def from_cimported_int_array():
+    """
+    >>> from_cimported_int_array()
+    [1, 2, 3]
+    """
+    cdef stdint.int32_t[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
+    return v
+
+
+def from_cimported_as_int_array():
+    """
+    >>> from_cimported_as_int_array()
+    [1, 2, 3]
+    """
+    cdef my_int16_t[3] v
+    v[0] = 1
+    v[1] = 2
+    v[2] = 3
     return v
 
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/carrays.pyx cython-0.20.1+1~202203241016-9537/tests/run/carrays.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/carrays.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/carrays.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -14,6 +14,23 @@
     return x[0]
 
 
+def assign_index_in_loop():
+    """
+    >>> assign_index_in_loop()
+    2
+    """
+    cdef int i = 0
+    cdef int[1] a
+    cdef int[1] b
+    for a[0], b[0] in enumerate(range(3)):
+        assert a[0] == b[0]
+        assert a[0] == i
+        i += 1
+
+    assert a[0] == b[0]
+    return b[0]
+
+
 def test2():
     """
     >>> test2()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cascadedassignment.pyx cython-0.20.1+1~202203241016-9537/tests/run/cascadedassignment.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cascadedassignment.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cascadedassignment.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -56,3 +56,15 @@
     """
     a = b = c = float(expr())
     return a, b, c
+
+
+def test_overwrite():
+    """
+    >>> test_overwrite()
+    {0: {1: {2: {}}}}
+    """
+    x = a = {}
+    for i in range(3):
+        a[i] = a = {}
+    assert a == {}
+    return x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cascaded_list_unpacking_T467.pyx cython-0.20.1+1~202203241016-9537/tests/run/cascaded_list_unpacking_T467.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cascaded_list_unpacking_T467.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cascaded_list_unpacking_T467.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 467
+# ticket: t467
 
 def simple_parallel_assignment_from_call():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cascaded_typed_assignments_T466.pyx cython-0.20.1+1~202203241016-9537/tests/run/cascaded_typed_assignments_T466.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cascaded_typed_assignments_T466.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cascaded_typed_assignments_T466.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 466
+# ticket: t466
 # extension to T409
 
 cimport cython
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cascmp.pyx cython-0.20.1+1~202203241016-9537/tests/run/cascmp.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cascmp.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cascmp.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,38 @@
+# mode: run
+# tag: cascade, compare
+
+def ints_and_objects():
+    """
+    >>> ints_and_objects()
+    (0, 1, 0, 1, 1, 0)
+    """
+    cdef int int1=0, int2=0, int3=0, int4=0
+    cdef int r1, r2, r3, r4, r5, r6
+    cdef object obj1, obj2, obj3, obj4
+    obj1 = 1
+    obj2 = 2
+    obj3 = 3
+    obj4 = 4
+    r1 = int1 < int2 < int3
+    r2 = obj1 < obj2 < obj3
+    r3 = int1 < int2 < obj3
+    r4 = obj1 < 2 < 3
+    r5 = obj1 < 2 < 3 < 4
+    r6 = int1 < (int2 == int3) < int4
+    return r1, r2, r3, r4, r5, r6
+
+
+def const_cascade(x):
+    """
+    >>> const_cascade(2)
+    (True, False, True, False, False, True, False)
+    """
+    return (
+        0 <= 1,
+        1 <= 0,
+        1 <= 1 <= 2,
+        1 <= 0 < 1,
+        1 <= 1 <= 0,
+        1 <= 1 <= x <= 2 <= 3 > x <= 2 <= 2,
+        1 <= 1 <= x <= 1 <= 1 <= x <= 2,
+    )
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cclass_assign_attr_GH3100.pyx cython-0.20.1+1~202203241016-9537/tests/run/cclass_assign_attr_GH3100.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cclass_assign_attr_GH3100.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cclass_assign_attr_GH3100.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,19 @@
+cdef class Foo:
+    """
+    >>> D = Foo.__dict__
+    >>> D["meth"] is D["meth2"]
+    True
+    >>> D["classmeth"] is D["classmeth2"]
+    True
+    >>> D["staticmeth"] is D["staticmeth2"]
+    True
+    """
+    def meth(self): pass
+    @classmethod
+    def classmeth(cls): pass
+    @staticmethod
+    def staticmeth(): pass
+
+    meth2 = meth
+    classmeth2 = classmeth
+    staticmeth2 = staticmeth
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_bool_T227.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_bool_T227.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_bool_T227.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_bool_T227.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 227
+# ticket: t227
 
 from cpython.bool cimport bool
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_dataclass.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_dataclass.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_dataclass.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_dataclass.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,243 @@
+# mode: run
+# tag: dataclass
+
+from cython cimport dataclasses
+from cython.dataclasses cimport dataclass, field
+try:
+    import typing
+    from typing import ClassVar
+    from dataclasses import InitVar
+    import dataclasses as py_dataclasses
+except ImportError:
+    pass
+import cython
+from libc.stdlib cimport malloc, free
+
+include "../testsupport/cythonarrayutil.pxi"
+
+cdef class NotADataclass:
+    cdef cython.int a
+    b: float
+
+    def __repr__(self):
+        return "NADC"
+
+    def __str__(self):
+        return "string of NotADataclass"  # should be called - repr is called!
+
+    def __eq__(self, other):
+        return type(self) == type(other)
+
+    def __hash__(self):
+        return 1
+
+@dataclass(unsafe_hash=True)
+cdef class BasicDataclass:
+    """
+    >>> sorted(list(BasicDataclass.__dataclass_fields__.keys()))
+    ['a', 'b', 'c', 'd']
+
+    # Check the field type attribute - this is currently a string since
+    # it's taken from the annotation, but if we drop PEP563 in future
+    # then it may change
+    >>> BasicDataclass.__dataclass_fields__["a"].type
+    'float'
+    >>> BasicDataclass.__dataclass_fields__["b"].type
+    'NotADataclass'
+    >>> BasicDataclass.__dataclass_fields__["c"].type
+    'object'
+    >>> BasicDataclass.__dataclass_fields__["d"].type
+    'list'
+
+    >>> inst1 = BasicDataclass() # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: __init__() takes at least 1 ...
+    >>> inst1 = BasicDataclass(2.0)
+
+    # The error at-least demonstrates that the hash function has been created
+    >>> hash(inst1) # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...unhashable...
+    >>> inst2 = BasicDataclass(2.0)
+    >>> inst1 == inst2
+    True
+    >>> inst2 = BasicDataclass(2.0, NotADataclass(), [])
+    >>> inst1 == inst2
+    False
+    >>> inst2 = BasicDataclass(2.0, NotADataclass(), [], [1,2,3])
+    >>> inst2
+    BasicDataclass(a=2.0, b=NADC, c=[], d=[1, 2, 3])
+    >>> inst2.c = "Some string"
+    >>> inst2
+    BasicDataclass(a=2.0, b=NADC, c='Some string', d=[1, 2, 3])
+    """
+    a: float
+    b: NotADataclass = field(default_factory=NotADataclass)
+    c: object = field(default=0)
+    d: list = dataclasses.field(default_factory=list)
+
+@dataclasses.dataclass
+cdef class InheritsFromDataclass(BasicDataclass):
+    """
+    >>> sorted(list(InheritsFromDataclass.__dataclass_fields__.keys()))
+    ['a', 'b', 'c', 'd', 'e']
+    >>> InheritsFromDataclass(a=1.0, e=5)
+    In __post_init__
+    InheritsFromDataclass(a=1.0, b=NADC, c=0, d=[], e=5)
+    """
+    e: cython.int = 0
+
+    def __post_init__(self):
+        print "In __post_init__"
+
+@cython.dataclasses.dataclass
+cdef class InheritsFromNotADataclass(NotADataclass):
+    """
+    >>> sorted(list(InheritsFromNotADataclass.__dataclass_fields__.keys()))
+    ['c']
+    >>> InheritsFromNotADataclass()
+    InheritsFromNotADataclass(c=1)
+    >>> InheritsFromNotADataclass(5)
+    InheritsFromNotADataclass(c=5)
+    """
+
+    c: cython.int = 1
+
+cdef struct S:
+    int a
+
+ctypedef S* S_ptr
+
+cdef S_ptr malloc_a_struct():
+    return malloc(sizeof(S))
+
+@dataclass
+cdef class ContainsNonPyFields:
+    """
+    >>> ContainsNonPyFields()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: __init__() takes ... 1 positional ...
+    >>> ContainsNonPyFields(mystruct={'a': 1 })  # doctest: +ELLIPSIS
+    ContainsNonPyFields(mystruct={'a': 1}, memview=)
+    >>> ContainsNonPyFields(mystruct={'a': 1 }, memview=create_array((2,2), "c"))  # doctest: +ELLIPSIS
+    ContainsNonPyFields(mystruct={'a': 1}, memview=)
+    >>> ContainsNonPyFields(mystruct={'a': 1 }, mystruct_ptr=0)
+    Traceback (most recent call last):
+    TypeError: __init__() got an unexpected keyword argument 'mystruct_ptr'
+    """
+    mystruct: S = cython.dataclasses.field(compare=False)
+    mystruct_ptr: S_ptr = field(init=False, repr=False, default_factory=malloc_a_struct)
+    memview: int[:, ::1] = field(default=create_array((3,1), "c"),  # mutable so not great but OK for a test
+                                 compare=False)
+
+    def __dealloc__(self):
+        free(self.mystruct_ptr)
+
+@dataclass
+cdef class InitClassVars:
+    """
+    Private (i.e. defined with "cdef") members deliberately don't appear
+    TODO - ideally c1 and c2 should also be listed here
+    >>> sorted(list(InitClassVars.__dataclass_fields__.keys()))
+    ['a', 'b1', 'b2']
+    >>> InitClassVars.c1
+    2.0
+    >>> InitClassVars.e1
+    []
+    >>> inst1 = InitClassVars()
+    In __post_init__
+    >>> inst1  # init vars don't appear in string
+    InitClassVars(a=0)
+    >>> inst2 = InitClassVars(b1=5, d2=100)
+    In __post_init__
+    >>> inst1 == inst2  # comparison ignores the initvar
+    True
+    """
+    a: cython.int = 0
+    b1: InitVar[double] = 1.0
+    b2: py_dataclasses.InitVar[double] = 1.0
+    c1: ClassVar[float] = 2.0
+    c2: typing.ClassVar[float] = 2.0
+    cdef InitVar[cython.int] d1
+    cdef py_dataclasses.InitVar[cython.int] d2
+    d1 = 5
+    d2 = 5
+    cdef ClassVar[list] e1
+    cdef typing.ClassVar[list] e2
+    e1 = []
+    e2 = []
+
+    def __post_init__(self, b1, b2, d1, d2):
+         # Check that the initvars haven't been assigned yet
+        assert self.b1==0, self.b1
+        assert self.b2==0, self.b2
+        assert self.d1==0, self.d1
+        assert self.d2==0, self.d2
+        self.b1 = b1
+        self.b2 = b2
+        self.d1 = d1
+        self.d2 = d2
+        print "In __post_init__"
+
+@dataclass
+cdef class TestVisibility:
+    """
+    >>> inst = TestVisibility()
+    >>> "a" in TestVisibility.__dataclass_fields__
+    False
+    >>> hasattr(inst, "a")
+    False
+    >>> "b" in TestVisibility.__dataclass_fields__
+    True
+    >>> hasattr(inst, "b")
+    True
+    >>> "c" in TestVisibility.__dataclass_fields__
+    True
+    >>> TestVisibility.__dataclass_fields__["c"].type
+    'double'
+    >>> hasattr(inst, "c")
+    True
+    """
+    cdef double a
+    a = 1.0
+    b: double = 2.0
+    cdef public double c
+    c = 3.0
+
+@dataclass(frozen=True)
+cdef class TestFrozen:
+    """
+    >>> inst = TestFrozen(a=5)
+    >>> inst.a
+    5.0
+    >>> inst.a = 2.  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    AttributeError: attribute 'a' of '...TestFrozen' objects is not writable
+    """
+    a: double = 2.0
+
+import sys
+if sys.version_info >= (3, 7):
+    __doc__ = """
+    >>> from dataclasses import Field, is_dataclass, fields
+
+    # It uses the types from the standard library where available
+    >>> all(isinstance(v, Field) for v in BasicDataclass.__dataclass_fields__.values())
+    True
+
+    # check out Cython dataclasses are close enough to convince it
+    >>> is_dataclass(BasicDataclass)
+    True
+    >>> is_dataclass(BasicDataclass(1.5))
+    True
+    >>> is_dataclass(InheritsFromDataclass)
+    True
+    >>> is_dataclass(NotADataclass)
+    False
+    >>> is_dataclass(InheritsFromNotADataclass)
+    True
+    >>> [ f.name for f in fields(BasicDataclass)]
+    ['a', 'b', 'c', 'd']
+    >>> [ f.name for f in fields(InitClassVars)]
+    ['a']
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_field.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_field.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_field.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_field.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: exttype
-# ticket: 677
+# ticket: t677
 
 """
 >>> str(Foo(4))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_property_decorator_T264.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_property_decorator_T264.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_class_property_decorator_T264.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_class_property_decorator_T264.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 264
+# ticket: t264
 # tag: property, decorator
 
 my_property = property
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_decorator_directives_T183.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_decorator_directives_T183.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_decorator_directives_T183.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_decorator_directives_T183.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 183
+# ticket: t183
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_locals_decorator_T477.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_locals_decorator_T477.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_locals_decorator_T477.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_locals_decorator_T477.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 477
+# ticket: t477
 
 import cython
 @cython.locals(x=double)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_members_T517.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_members_T517.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_members_T517.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_members_T517.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 517
+# ticket: t517
 #cython: embedsignature=True
 
 __doc__ = u"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_methods_T462.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_methods_T462.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_methods_T462.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_methods_T462.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 462
+# ticket: t462
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_cimport.srctree cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_cimport.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_cimport.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_cimport.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,44 @@
+# Test for https://github.com/cython/cython/issues/4106
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import sub"
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("*.pyx"),
+)
+
+######## base.pxd ########
+
+cdef class A:
+  cdef dict __dict__
+  cdef int a(self)
+
+cdef class B(A):
+  cdef int b(self)
+
+######## base.pyx ########
+
+cdef class A:
+  cdef int a(self):
+    return 1
+
+class PyA:
+  pass
+
+cdef class B(A, PyA):
+  cdef int b(self):
+    return 2
+
+######## sub.pyx ########
+
+from base cimport B
+print(B)
+
+cdef class C(B):
+  cdef int c(self):
+    return 3
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_errors.srctree cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_errors.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_errors.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_errors.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,95 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(ext_modules=cythonize("*.pyx"))
+
+######## notheaptype.pyx ########
+
+cdef class Base:
+    pass
+
+Obj = type(object())
+
+cdef class Foo(Base, Obj):
+    pass
+
+######## wrongbase.pyx ########
+
+cdef class Base:
+    pass
+
+Str = type("")
+
+cdef class X(Base, Str):
+    pass
+
+######## badmro.pyx ########
+
+class Py(object):
+    pass
+
+cdef class X(object, Py):
+    pass
+
+######## nodict.pyx ########
+
+cdef class Base:
+    pass
+
+class Py(object):
+    pass
+
+cdef class X(Base, Py):
+    pass
+
+######## oldstyle.pyx ########
+# cython: language_level=2
+
+cdef class Base:
+    cdef dict __dict__
+
+class OldStyle:
+    pass
+
+cdef class Foo(Base, OldStyle):
+    pass
+
+######## runner.py ########
+
+import sys
+
+try:
+    import notheaptype
+    assert False, "notheaptype"
+except TypeError as msg:
+    assert str(msg) == "base class 'object' is not a heap type"
+
+try:
+    import wrongbase
+    assert False, "wrongbase"
+except TypeError as msg:
+    assert str(msg) == "best base 'str' must be equal to first base 'wrongbase.Base'"
+
+try:
+    import badmro
+    assert False, "badmro"
+except TypeError as msg:
+    assert str(msg).startswith("Cannot create a consistent method resolution")
+
+try:
+    import nodict
+    assert False, "nodict"
+except TypeError as msg:
+    assert str(msg) == "extension type 'nodict.X' has no __dict__ slot, but base type 'Py' has: either add 'cdef dict __dict__' to the extension type or add '__slots__ = [...]' to the base type"
+
+try:
+    # This should work on Python 3 but fail on Python 2
+    import oldstyle
+    assert sys.version_info[0] >= 3, "oldstyle"
+except TypeError as msg:
+    assert str(msg) == "base class 'OldStyle' is an old-style class"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_nodict.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_nodict.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance_nodict.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance_nodict.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,48 @@
+# Copied from cdef_multiple_inheritance.pyx
+# but with __slots__ and without __dict__
+
+cdef class CBase(object):
+    cdef int a
+    cdef c_method(self):
+        return "CBase"
+    cpdef cpdef_method(self):
+        return "CBase"
+
+class PyBase(object):
+    __slots__ = []
+    def py_method(self):
+        return "PyBase"
+
+cdef class Both(CBase, PyBase):
+    """
+    >>> b = Both()
+    >>> b.py_method()
+    'PyBase'
+    >>> b.cp_method()
+    'Both'
+    >>> b.call_c_method()
+    'Both'
+
+    >>> isinstance(b, CBase)
+    True
+    >>> isinstance(b, PyBase)
+    True
+    """
+    cdef c_method(self):
+        return "Both"
+    cpdef cp_method(self):
+        return "Both"
+    def call_c_method(self):
+        return self.c_method()
+
+cdef class BothSub(Both):
+    """
+    >>> b = BothSub()
+    >>> b.py_method()
+    'PyBase'
+    >>> b.cp_method()
+    'Both'
+    >>> b.call_c_method()
+    'Both'
+    """
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_multiple_inheritance.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_multiple_inheritance.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,72 @@
+cimport cython
+
+cdef class CBase(object):
+    cdef int a
+    cdef c_method(self):
+        return "CBase"
+    cpdef cpdef_method(self):
+        return "CBase"
+
+class PyBase(object):
+    def py_method(self):
+        return "PyBase"
+
+@cython.binding(True)
+cdef class BothBound(CBase, PyBase):
+    cdef dict __dict__
+    """
+    >>> b = Both()
+    >>> b.py_method()
+    'PyBase'
+    >>> b.cp_method()
+    'Both'
+    >>> b.call_c_method()
+    'Both'
+
+    >>> isinstance(b, CBase)
+    True
+    >>> isinstance(b, PyBase)
+    True
+    """
+    cdef c_method(self):
+        return "Both"
+    cpdef cp_method(self):
+        return "Both"
+    def call_c_method(self):
+        return self.c_method()
+
+cdef class BothSub(BothBound):
+    """
+    >>> b = BothSub()
+    >>> b.py_method()
+    'PyBase'
+    >>> b.cp_method()
+    'Both'
+    >>> b.call_c_method()
+    'Both'
+    """
+    pass
+
+@cython.binding(False)
+cdef class BothUnbound(CBase, PyBase):
+    cdef dict __dict__
+    """
+    >>> b = Both()
+    >>> b.py_method()
+    'PyBase'
+    >>> b.cp_method()
+    'Both'
+    >>> b.call_c_method()
+    'Both'
+
+    >>> isinstance(b, CBase)
+    True
+    >>> isinstance(b, PyBase)
+    True
+    """
+    cdef c_method(self):
+        return "Both"
+    cpdef cp_method(self):
+        return "Both"
+    def call_c_method(self):
+        return self.c_method()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdef_setitem_T284.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdef_setitem_T284.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdef_setitem_T284.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdef_setitem_T284.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 284
+# ticket: t284
 
 def no_cdef():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cdivision_CEP_516.pyx cython-0.20.1+1~202203241016-9537/tests/run/cdivision_CEP_516.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cdivision_CEP_516.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cdivision_CEP_516.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -27,6 +27,9 @@
 >>> [test_cdiv_cmod(a, b) for a, b in v]
 [(1, 7), (-1, -7), (1, -7), (-1, 7)]
 
+>>> [test_cdiv_cmod(a, b) for a, b in [(4, -4), (4, -2), (4, -1)]]
+[(-1, 0), (-2, 0), (-4, 0)]
+
 >>> all([mod_int_py(a,b) == a % b for a in range(-10, 10) for b in range(-10, 10) if b != 0])
 True
 >>> all([div_int_py(a,b) == a // b for a in range(-10, 10) for b in range(-10, 10) if b != 0])
@@ -191,3 +194,14 @@
     OverflowError: ...
     """
     return a / b
+
+def c_div_const_test(a, b):
+    """
+    >>> c_div_const_test(5, 3)
+    1
+    """
+    return c_div_const(a, b)
+
+cdef long c_div_const(const long a, int b):
+    cdef long c = a / b
+    return c
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cfunc_call_tuple_args_T408.pyx cython-0.20.1+1~202203241016-9537/tests/run/cfunc_call_tuple_args_T408.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cfunc_call_tuple_args_T408.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cfunc_call_tuple_args_T408.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 408
+# ticket: t408
 
 __doc__ = """
 >>> call_with_tuple(1, 1.2, 'test', [1,2,3])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cfunc_convert.pyx cython-0.20.1+1~202203241016-9537/tests/run/cfunc_convert.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cfunc_convert.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cfunc_convert.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,5 @@
 # mode: run
+# tag: autowrap
 # cython: always_allow_keywords=True
 
 cimport cython
@@ -82,7 +83,7 @@
 
 cdef long long rad(long long x):
     cdef long long rad = 1
-    for p in range(2, sqrt(x) + 1):
+    for p in range(2, sqrt(x) + 1):  # MSVC++ fails without the input cast
         if x % p == 0:
             rad *= p
             while x % p == 0:
@@ -229,3 +230,39 @@
     TypeError: Argument 'b' has incorrect type (expected cfunc_convert.B, got cfunc_convert.A)
     """
     return (test_cdef_class_params_cfunc)(a, b)
+
+# There were a few cases where duplicate utility code definitions (i.e. with the same name)
+# could be generated, causing C compile errors. This file tests them.
+
+cdef cfunc_dup_f1(x, r):
+    return "f1"
+
+cdef cfunc_dup_f2(x1, r):
+    return "f2"
+
+def make_map():
+    """
+    https://github.com/cython/cython/issues/3716
+    This is testing the generation of wrappers for f1 and f2
+    >>> for k, f in make_map().items():
+    ...    print(k == f(0, 0))  # in both cases the functions should just return their name
+    True
+    True
+
+    # Test passing of keyword arguments
+    >>> print(make_map()['f1'](x=1, r=2))
+    f1
+    >>> make_map()['f1'](x1=1, r=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    >>> print(make_map()['f2'](x1=1, r=2))
+    f2
+    >>> make_map()['f2'](x=1, r=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    """
+    cdef map = {
+        "f1": cfunc_dup_f1,
+        "f2": cfunc_dup_f2,
+    }
+    return map
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cfuncptr.pyx cython-0.20.1+1~202203241016-9537/tests/run/cfuncptr.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cfuncptr.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cfuncptr.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,41 @@
+# mode: run
+
+
+cdef int grail():
+    cdef int (*spam)()
+    spam = &grail
+    spam = grail
+    assert spam is grail
+    assert spam == grail
+    assert spam == &grail
+
+
+ctypedef int funcptr_t()
+
+cdef funcptr_t* get_grail():
+    return &grail
+
+
+def test_assignments():
+    """
+    >>> test_assignments()
+    """
+    grail()
+
+
+def test_return_value():
+    """
+    >>> test_return_value()
+    True
+    """
+    g = get_grail()
+    return g == &grail
+
+
+def call_cfuncptr():
+    """
+    >>> call_cfuncptr()
+    """
+    cdef int (*spam)()
+    spam = grail
+    spam()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/charcomparisonT412.pyx cython-0.20.1+1~202203241016-9537/tests/run/charcomparisonT412.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/charcomparisonT412.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/charcomparisonT412.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 412
+# ticket: t412
 
 def f():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/char_constants_T99.pyx cython-0.20.1+1~202203241016-9537/tests/run/char_constants_T99.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/char_constants_T99.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/char_constants_T99.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 99
+# ticket: t99
 
 cdef char c = 'c'
 cdef char* s = 'abcdef'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/charptr_comparison_T582.pyx cython-0.20.1+1~202203241016-9537/tests/run/charptr_comparison_T582.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/charptr_comparison_T582.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/charptr_comparison_T582.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 582
+# ticket: t582
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/charptr_decode.pyx cython-0.20.1+1~202203241016-9537/tests/run/charptr_decode.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/charptr_decode.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/charptr_decode.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,11 @@
 
 cimport cython
 
+cdef extern from *:
+    cdef Py_ssize_t PY_SSIZE_T_MIN
+    cdef Py_ssize_t PY_SSIZE_T_MAX
+
+
 ############################################################
 # tests for char* slicing
 
@@ -118,6 +123,19 @@
             (cstring+1)[:].decode('UTF-8'),
             (cstring+1)[return1():return5()].decode('UTF-8'))
 
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+@cython.test_fail_if_path_exists("//AttributeNode")
+def slice_charptr_decode_large_bounds():
+    """
+    >>> print(str(slice_charptr_decode_large_bounds()).replace("u'", "'"))
+    ('abcABCqtp', '', '', '')
+    """
+    return (cstring[PY_SSIZE_T_MIN:9].decode('UTF-8'),
+            cstring[PY_SSIZE_T_MAX:PY_SSIZE_T_MIN].decode('UTF-8'),
+            cstring[PY_SSIZE_T_MIN:PY_SSIZE_T_MIN].decode('UTF-8'),
+            cstring[PY_SSIZE_T_MAX:PY_SSIZE_T_MAX].decode('UTF-8'))
+
+
 cdef return1(): return 1
 cdef return3(): return 3
 cdef return4(): return 4
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/check_size.srctree cython-0.20.1+1~202203241016-9537/tests/run/check_size.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/check_size.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/check_size.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,234 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from Cython.Compiler.Errors import CompileError
+from distutils.core import setup
+
+# force the build order
+setup(ext_modules= cythonize("check_size.pyx"))
+
+setup(ext_modules = cythonize("_check_size*.pyx"))
+
+try:
+    setup(ext_modules= cythonize("check_size_invalid.pyx"))
+    assert False
+except CompileError as e:
+    pass
+
+######## check_size_nominal.h ########
+
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f0;
+    int f1;
+    int f2;
+} FooStructNominal;
+
+#ifdef __cplusplus
+}
+#endif
+
+######## check_size_bigger.h ########
+
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f0;
+    int f1;
+    int f2;
+    int f3;
+    int f4;
+} FooStructBig;
+
+#ifdef __cplusplus
+}
+#endif
+
+######## check_size_smaller.h ########
+
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f9;
+} FooStructSmall;
+
+#ifdef __cplusplus
+}
+#endif
+
+
+######## check_size.pyx ########
+
+cdef class Foo:
+    cdef public int field0, field1, field2;
+
+    def __init__(self, f0, f1, f2):
+        self.field0 = f0
+        self.field1 = f1
+        self.field2 = f2
+
+######## _check_size_exact.pyx ########
+
+cdef extern from "check_size_nominal.h":
+
+    ctypedef class check_size.Foo [object FooStructNominal]:
+        cdef:
+            int f0
+            int f1
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f0 + f.f1
+
+######## _check_size_too_small.pyx ########
+
+cdef extern from "check_size_bigger.h":
+
+    ctypedef class check_size.Foo [object FooStructBig]:
+        cdef:
+            int f0
+            int f1
+            int f2
+
+
+cpdef public int testme(Foo f, int f2) except -1:
+    f.f2 = f2
+    return f.f0 + f.f1 + f.f2
+
+######## _check_size_default.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    ctypedef class check_size.Foo [object FooStructSmall]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## _check_size_warn.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    # make sure missing check_size is equivalent to warn
+    ctypedef class check_size.Foo [object FooStructSmall, check_size warn]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## _check_size_ignore.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    # Allow size to be larger
+    ctypedef class check_size.Foo [object FooStructSmall, check_size ignore]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## _check_size_error.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    # Strict checking, will raise an error
+    ctypedef class check_size.Foo [object FooStructSmall, check_size error]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## check_size_invalid.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    # Raise CompileError when using bad value
+    ctypedef class check_size.Foo [object FooStructSmall, check_size hihi]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## runner.py ########
+
+import check_size, _check_size_exact, warnings
+
+foo = check_size.Foo(23, 123, 1023)
+
+assert foo.field0 == 23
+assert foo.field1 == 123
+
+ret =  _check_size_exact.testme(foo)
+assert ret == 23 + 123
+
+# ValueError since check_size.Foo's tp_basicsize is smaller than what is needed
+# for FooStructBig. Messing with f2 will access memory outside the struct!
+try:
+    import _check_size_too_small
+    assert False
+except ValueError as e:
+    assert str(e).startswith('check_size.Foo size changed')
+
+# Warining since check_size.Foo's tp_basicsize is larger than what is needed
+# for FooStructSmall. There is "spare", accessing FooStructSmall's fields will
+# never access invalid memory. This can happen, for instance, when using old
+# headers with a newer runtime, or when using an old _check_size{2,3} with a newer
+# check_size, where the developers of check_size are careful to be backward
+# compatible.
+
+with warnings.catch_warnings(record=True) as w:
+    warnings.simplefilter("always")
+    import _check_size_default
+    import _check_size_warn
+    assert len(w) == 2, 'expected two warnings, got %d' % len(w)
+    assert str(w[0].message).startswith('check_size.Foo size changed')
+    assert str(w[1].message).startswith('check_size.Foo size changed')
+
+ret = _check_size_default.testme(foo)
+assert ret == 23
+ret = _check_size_warn.testme(foo)
+assert ret == 23
+
+with warnings.catch_warnings(record=True) as w:
+    # No warning, runtime vendor must provide backward compatibility
+    import _check_size_ignore
+    assert len(w) == 0
+
+ret = _check_size_ignore.testme(foo)
+assert ret == 23
+
+try:
+    # Enforce strict checking
+    import _check_size_error
+    assert False
+except ValueError as e:
+    assert str(e).startswith('check_size.Foo size changed')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cimport_cython_T505.pyx cython-0.20.1+1~202203241016-9537/tests/run/cimport_cython_T505.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cimport_cython_T505.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cimport_cython_T505.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 505
+# ticket: t505
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cimport_from_pyx.srctree cython-0.20.1+1~202203241016-9537/tests/run/cimport_from_pyx.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cimport_from_pyx.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cimport_from_pyx.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -15,14 +15,22 @@
 
 ######## a.pyx ########
 
-from b cimport Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE
+from b cimport (Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE,
+                DecoratedClass, cfuncOutside)
 cdef Bclass b = Bclass(5)
 assert Bfunc(&b.value) == b.value
+assert b.anotherValue == 6, b.anotherValue
 assert b.asStruct().value == b.value
 cdef Btypedef b_type = &b.value
 cdef Benum b_enum = Benum_value
 cdef int tmp = Py_EQ
 
+cdef DecoratedClass dc = DecoratedClass()
+assert dc.cfuncInClass().value == 5
+assert dc.cpdefInClass() == 1.0
+
+assert cfuncOutside().value == 2
+
 #from c cimport ClassC
 #cdef ClassC c = ClassC()
 #print c.value
@@ -31,6 +39,8 @@
 
 from cpython.object cimport Py_EQ, Py_NE
 
+cimport cython
+
 cdef enum Benum:
     Benum_value
 
@@ -41,14 +51,34 @@
 
 cdef class Bclass:
     cdef long value
+    anotherValue: cython.double
     def __init__(self, value):
         self.value = value
+        self.anotherValue = value + 1
     cdef Bstruct asStruct(self):
         return Bstruct(value=self.value)
+    cdef double getOtherValue(self):
+        return self.anotherValue
 
 cdef long Bfunc(Btypedef x):
     return x[0]
 
+@cython.cclass
+class DecoratedClass:
+    @cython.cfunc
+    @cython.returns(Bstruct)
+    def cfuncInClass(self):
+        return Bstruct(value=5)
+    @cython.ccall
+    @cython.returns(cython.double)
+    def cpdefInClass(self):
+        return 1.0
+
+@cython.cfunc
+@cython.returns(Bstruct)
+def cfuncOutside():
+    return Bstruct(value=2)
+
 ######## c.pxd ########
 
 cdef class ClassC:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cimport_from_sys_path.srctree cython-0.20.1+1~202203241016-9537/tests/run/cimport_from_sys_path.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cimport_from_sys_path.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cimport_from_sys_path.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -19,13 +19,20 @@
 
 ######## site-packages/b/other.pxd ########
 
-cdef inline foo(int a):
-     return a**2
+cdef extern from "foo.c":
+    int foo(int)
+
+######## site-packages/b/foo.c ########
+
+static int foo(int a)
+{
+    return a * a;
+}
 
 ######## a.pyx ########
 
 from b.other cimport foo
-print foo(10)
+print(foo(10))
 
 cimport b.other
-print b.other.foo(10)
+print(b.other.foo(10))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cimport.srctree cython-0.20.1+1~202203241016-9537/tests/run/cimport.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cimport.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cimport.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -26,10 +26,39 @@
 cdef int foo(int a):
      return a**2
 
+######## pkg/__init__.py ########
+
+
+######## pkg/sub.pxd ########
+
+ctypedef int my_int
+
+######## pkg/subpkg/__init__.py ########
+
+######## pkg/subpkg/submod.pxd ########
+
+ctypedef int my_int
+
 ######## a.pyx ########
 
-from other cimport A, foo
-print A, foo(10)
+from other cimport (
+    A,
+    foo,
+)
+print(A, foo(10))
 
 cimport other
-print other.A, other.foo(10)
+
+cdef call_fooptr(int (*fptr)(int)):
+    return fptr(10)
+
+def call_other_foo():
+    x = other.foo  # GH4000 - failed because other was untyped
+    return call_fooptr(x) # check that x is correctly resolved as a function pointer
+
+print(other.A, other.foo(10), call_other_foo())
+
+from pkg cimport sub
+cdef sub.my_int a = 100
+
+from pkg.subpkg cimport submod
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cintop.pyx cython-0.20.1+1~202203241016-9537/tests/run/cintop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cintop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cintop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,5 @@
+# mode: run
+
 __doc__ = u"""
     >>> int2 = 42
     >>> int3 = 7
@@ -35,3 +37,24 @@
     int1 ^= int2 << int3 | int2 >> int3
     long1 = char1 | int1
     return int1, long1
+
+
+def long_int_shift():
+    """
+    >>> long_int_shift()
+    80082
+    10010
+    10010
+    10010
+    10010
+    """
+    value = 80082 # int using more than 2 bytes == long
+    print(value)
+    shiftedby3 = value >> 3
+    dividedby8 = value // 8
+    print(shiftedby3)
+    print(dividedby8)
+    shiftedby3 = 80082 >> 3
+    dividedby8 = 80082 // 8
+    print(shiftedby3)
+    print(dividedby8)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/c_int_types_T255.pyx cython-0.20.1+1~202203241016-9537/tests/run/c_int_types_T255.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/c_int_types_T255.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/c_int_types_T255.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 255
+# ticket: t255
 
 __doc__ = u""
 
@@ -685,14 +685,13 @@
 
 def test_convert_pyint(x):
    u"""
-   >>> test_convert_pyint(None)
+   >>> test_convert_pyint(None)  # doctest: +ELLIPSIS
    Traceback (most recent call last):
-       ...
-   TypeError: an integer is required
-   >>> test_convert_pyint("123")
+   TypeError:... int...
+   >>> test_convert_pyint("123")  # doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
-   TypeError: an integer is required
+   TypeError:... int...
    >>> test_convert_pyint(MyBadInt(0)) #doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
@@ -733,14 +732,14 @@
 
 def test_convert_pylong(x):
    u"""
-   >>> test_convert_pylong(None)
+   >>> test_convert_pylong(None)  # doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
-   TypeError: an integer is required
-   >>> test_convert_pylong("123")
+   TypeError:... int...
+   >>> test_convert_pylong("123")  # doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
-   TypeError: an integer is required
+   TypeError:... int...
    >>> test_convert_pylong(MyBadLong(0)) #doctest: +ELLIPSIS
    Traceback (most recent call last):
        ...
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/class_attribute_init_values_T18.pyx cython-0.20.1+1~202203241016-9537/tests/run/class_attribute_init_values_T18.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/class_attribute_init_values_T18.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/class_attribute_init_values_T18.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 18
+# ticket: t18
 
 __doc__ = u"""
 >>> f = PyFoo()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/classdecorators_T336.pyx cython-0.20.1+1~202203241016-9537/tests/run/classdecorators_T336.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/classdecorators_T336.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/classdecorators_T336.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 336
+# ticket: t336
 
 __doc__ = u"""
 >>> print('\\n'.join(calls))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/class_func_in_control_structures_T87.pyx cython-0.20.1+1~202203241016-9537/tests/run/class_func_in_control_structures_T87.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/class_func_in_control_structures_T87.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/class_func_in_control_structures_T87.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 87
+# ticket: t87
 
 __doc__ = u"""
 >>> d = Defined()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/classmethod.pyx cython-0.20.1+1~202203241016-9537/tests/run/classmethod.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/classmethod.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/classmethod.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -9,6 +9,10 @@
 class1
 >>> class1().bview()
 class1
+>>> class1().cview()
+class1
+>>> class1().cview("XX")
+class1XX
 
 >>> class2.view()
 class2
@@ -35,6 +39,12 @@
 def f_plus(cls, a):
     return cls.a + a
 
+def second_decorator(f):
+    # note - a class, not a function (didn't pass Cython's test in __Pyx_Method_ClassMethod)
+    class C:
+        def __call__(self, *args):
+            return f(*args)
+    return C()
 
 class class1:
     a = 5
@@ -48,6 +58,11 @@
     def bview(cls):
         print cls.__name__
 
+    @classmethod
+    @second_decorator
+    def cview(cls, s=""):
+        print cls.__name__+s
+
 
 class class2(object):
     a = 6
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/class_scope_del_T684.py cython-0.20.1+1~202203241016-9537/tests/run/class_scope_del_T684.py
--- cython-0.20.1+1~201611251650-6686/tests/run/class_scope_del_T684.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/class_scope_del_T684.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode:run
 # tag: class, scope, del
-# ticket: 684
+# ticket: t684
 
 class DelInClass(object):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/clear_to_null.pyx cython-0.20.1+1~202203241016-9537/tests/run/clear_to_null.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/clear_to_null.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/clear_to_null.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,7 +2,7 @@
 Check that Cython generates a tp_clear function that actually clears object
 references to NULL instead of None.
 
-Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14833
+Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14833
 """
 
 from cpython.ref cimport PyObject, Py_TYPE
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_class_T596.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_class_T596.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_class_T596.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_class_T596.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 596
+# ticket: t596
 
 def simple(a, b):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_decorators_T478.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_decorators_T478.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_decorators_T478.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_decorators_T478.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 478
+# ticket: t478
 
 __doc__ = """
     >>> Num(13).is_prime()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_in_derived_class_T2967.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_in_derived_class_T2967.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_in_derived_class_T2967.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_in_derived_class_T2967.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,18 @@
+# mode: run
+# tag: closures
+# ticket: 2967
+
+cdef class BaseClass:
+    cdef func(self):
+        pass
+cdef class ClosureInsideExtensionClass(BaseClass):
+    """
+    >>> y = ClosureInsideExtensionClass(42)
+    >>> y.test(42)
+    43
+    """
+    cdef func(self):
+        a = 1
+        return (lambda x : x+a)
+    def test(self, b):
+        return self.func()(b)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_inside_cdef_T554.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_inside_cdef_T554.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_inside_cdef_T554.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_inside_cdef_T554.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 554
+# ticket: t554
 
 def call_f(x):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_name_mangling_T537.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_name_mangling_T537.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_name_mangling_T537.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_name_mangling_T537.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 537
+# ticket: t537
 
 __doc__ = u"""
 >>> f1 = nested1()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closure_names.pyx cython-0.20.1+1~202203241016-9537/tests/run/closure_names.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closure_names.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closure_names.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,116 @@
+# mode: run
+# tag: closures
+# ticket: 1797
+
+
+def func():
+    """
+    >>> funcs = func()
+    >>> [f(1) for f in funcs]  # doctest: +NORMALIZE_WHITESPACE
+    ['eq',
+     'str',
+     'weakref',
+     'new',
+     'getitem',
+     'setitem',
+     'delitem',
+     'getslice',
+     'setslice',
+     'delslice_',
+     'getattr',
+     'getattribute',
+     'setattr',
+     'delattr',
+     'get',
+     'set',
+     'delete',
+     'dict',
+     'dealloc',
+     'cinit']
+    """
+    def __eq__(a):
+        return 'eq'
+
+    def __str__(a):
+        return 'str'
+
+    def __weakref__(a):
+        return 'weakref'
+
+    def __new__(a):
+        return 'new'
+
+    def __getitem__(a):
+        return 'getitem'
+
+    def __setitem__(a):
+        return 'setitem'
+
+    def __delitem__(a):
+        return 'delitem'
+
+    def __getslice__(a):
+        return 'getslice'
+
+    def __setslice__(a):
+        return 'setslice'
+
+    def __delslice__(a):
+        return 'delslice_'
+
+    def __getattr__(a):
+        return 'getattr'
+
+    def __getattribute__(a):
+        return 'getattribute'
+
+    def __setattr__(a):
+        return 'setattr'
+
+    def __delattr__(a):
+        return 'delattr'
+
+    def __get__(a):
+        return 'get'
+
+    def __set__(a):
+        return 'set'
+
+    def __delete__(a):
+        return 'delete'
+
+    def __dict__(a):
+        return 'dict'
+
+    def __dealloc__(a):
+        return 'dealloc'
+
+    def __cinit__(a):
+        return 'cinit'
+
+    def list_from_gen(g):
+        return list(g)
+
+    # move into closure by using inside of generator expression
+    return list_from_gen([
+            __eq__,
+            __str__,
+            __weakref__,
+            __new__,
+            __getitem__,
+            __setitem__,
+            __delitem__,
+            __getslice__,
+            __setslice__,
+            __delslice__,
+            __getattr__,
+            __getattribute__,
+            __setattr__,
+            __delattr__,
+            __get__,
+            __set__,
+            __delete__,
+            __dict__,
+            __dealloc__,
+            __cinit__,
+        ][i] for i in range(20))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/closures_T82.pyx cython-0.20.1+1~202203241016-9537/tests/run/closures_T82.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/closures_T82.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/closures_T82.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 82
+# ticket: t82
 # preparse: id
 # preparse: def_to_cdef
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cmethod_inline_T474.pyx cython-0.20.1+1~202203241016-9537/tests/run/cmethod_inline_T474.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cmethod_inline_T474.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cmethod_inline_T474.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 474
+# ticket: t474
 cimport cython
 
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/common_utility_types.srctree cython-0.20.1+1~202203241016-9537/tests/run/common_utility_types.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/common_utility_types.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/common_utility_types.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -31,8 +31,21 @@
 import a, b
 print(type(a.funcA))
 
-assert type(a.funcA).__name__ == 'cython_function_or_method'
+assert type(a.funcA).__name__.endswith('cython_function_or_method')
 assert type(a.funcA) is type(b.funcB)
 
 assert a.funcA.func_globals is a.__dict__
 assert b.funcB.func_globals is b.__dict__
+
+# Test that it's possible to look up the name of the class
+from sys import modules
+cy_modules = [ mod for n, mod in modules.items() if n.startswith("_cython_") ]
+# In principle it's possible to have "_cython_" internal modules for multiple
+# different versions of Cython. However, since this is run in an end-to-end test
+# with a very short list of imports it should not happen here.
+assert(len(cy_modules)==1)
+mod = cy_modules[0]
+
+assert '.' not in type(a.funcA).__name__
+func_t = getattr(mod, type(a.funcA).__name__)
+assert func_t is type(a.funcA)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_cast_T445.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_cast_T445.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_cast_T445.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_cast_T445.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 445
+# ticket: t445
 
 def complex_double_cast(double x, double complex z):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_coercion_sideeffects_T693.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_coercion_sideeffects_T693.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_coercion_sideeffects_T693.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_coercion_sideeffects_T693.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 693
+# ticket: t693
 
 cdef double complex func(double complex x):                                                  
     print "hello"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_int_T446_fix.h cython-0.20.1+1~202203241016-9537/tests/run/complex_int_T446_fix.h
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_int_T446_fix.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_int_T446_fix.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,3 +0,0 @@
-#if defined _MSC_VER && defined __cplusplus
-#define CYTHON_CCOMPLEX 0
-#endif
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_int_T446.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_int_T446.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_int_T446.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_int_T446.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,9 +1,14 @@
-# ticket: 446
+# ticket: t446
 
 import cython
 
-cdef extern from "complex_int_T446_fix.h":
-    pass
+cdef extern from *:
+    """
+    #if defined _MSC_VER && defined __cplusplus
+    #define CYTHON_CCOMPLEX 0
+    #endif
+    """
+
 
 def test_arith(int complex a, int complex b):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c89_T398_long_double.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c89_T398_long_double.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c89_T398_long_double.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c89_T398_long_double.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,4 @@
+# ticket: t398
+
+cdef extern from "complex_numbers_c89_T398.h": pass
+include "complex_numbers_T305_long_double.pyx"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c89_T398.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c89_T398.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c89_T398.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c89_T398.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 398
+# ticket: t398
 
 cdef extern from "complex_numbers_c89_T398.h": pass
 include "complex_numbers_T305.pyx"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c99_T398.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c99_T398.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_c99_T398.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_c99_T398.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 398
+# ticket: t398
 
 cdef extern from "complex_numbers_c99_T398.h": pass
 include "complex_numbers_T305.pyx"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_cmath_T2891.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_cmath_T2891.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_cmath_T2891.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_cmath_T2891.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,15 @@
+# ticket: 2891
+# tag: c, no-cpp
+
+cdef extern from "complex_numbers_c99_T398.h": pass
+
+from libc.complex cimport cimag, creal, cabs, carg
+
+
+def test_decomposing(double complex z):
+    """
+    >>> test_decomposing(3+4j)
+    (3.0, 4.0, 5.0, 0.9272952180016122)
+    """
+
+    return (creal(z), cimag(z), cabs(z), carg(z))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_cxx_T398.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_cxx_T398.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_cxx_T398.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_cxx_T398.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 398
+# ticket: t398
 
 cdef extern from "complex_numbers_cxx_T398.h": pass
 include "complex_numbers_T305.pyx"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_T305_long_double.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_T305_long_double.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_T305_long_double.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_T305_long_double.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,13 @@
+# ticket: t305
+
+cimport cython
+
+def test_object_conversion(o):
+    """
+    >>> test_object_conversion(2)
+    (2+0j)
+    >>> test_object_conversion(2j - 0.5)
+    (-0.5+2j)
+    """
+    cdef long double complex a = o
+    return a
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_T305.pyx cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_T305.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/complex_numbers_T305.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/complex_numbers_T305.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,18 +1,46 @@
-# ticket: 305
+# ticket: t305
+
+from cpython.object cimport Py_EQ, Py_NE
 
 cimport cython
 
+DEF C21 = 2-1j
+
+
+cdef class Complex3j:
+    """
+    >>> Complex3j() == 3j
+    True
+    >>> Complex3j() == Complex3j()
+    True
+    >>> Complex3j() != 3j
+    False
+    >>> Complex3j() != 3
+    True
+    >>> Complex3j() != Complex3j()
+    False
+    """
+    def __richcmp__(a, b, int op):
+        if op == Py_EQ or op == Py_NE:
+            if isinstance(a, Complex3j):
+                eq = isinstance(b, Complex3j) or b == 3j
+            else:
+                eq = isinstance(b, Complex3j) and a == 3j
+            return eq if op == Py_EQ else not eq
+        return NotImplemented
+
+
 def test_object_conversion(o):
     """
     >>> test_object_conversion(2)
-    ((2+0j), (2+0j), (2+0j))
+    ((2+0j), (2+0j))
     >>> test_object_conversion(2j - 0.5)
-    ((-0.5+2j), (-0.5+2j), (-0.5+2j))
+    ((-0.5+2j), (-0.5+2j))
     """
     cdef float complex a = o
     cdef double complex b = o
-    cdef long double complex c = o
-    return (a, b, c)
+    return (a, b)
+
 
 def test_arithmetic(double complex z, double complex w):
     """
@@ -25,6 +53,7 @@
     """
     return +z, -z+0, z+w, z-w, z*w, z/w
 
+
 def test_div(double complex a, double complex b, expected):
     """
     >>> big = 2.0**1023
@@ -35,6 +64,7 @@
     if '_c99_' not in __name__:
         assert a / b == expected, (a / b, expected)
 
+
 def test_pow(double complex z, double complex w, tol=None):
     """
     Various implementations produce slightly different results...
@@ -48,12 +78,15 @@
     True
     >>> test_pow(complex(0.5, -.25), complex(3, 4), 1e-15)
     True
+    >>> test_pow(-0.5, 1j, tol=1e-15)
+    True
     """
     if tol is None:
         return z**w
     else:
         return abs(z**w / z ** w - 1) < tol
 
+
 def test_int_pow(double complex z, int n, tol=None):
     """
     >>> [test_int_pow(complex(0, 1), k, 1e-15) for k in range(-4, 5)]
@@ -62,12 +95,15 @@
     [True, True, True, True, True, True, True, True, True]
     >>> [test_int_pow(complex(2, 0.5), k, 1e-14) for k in range(0, 10)]
     [True, True, True, True, True, True, True, True, True, True]
+    >>> test_int_pow(-0.5, 5, tol=1e-15)
+    True
     """
     if tol is None:
         return z**n + 0 # add zero to normalize zero sign
     else:
         return abs(z**n / z ** n - 1) < tol
 
+
 @cython.cdivision(False)
 def test_div_by_zero(double complex z):
     """
@@ -80,6 +116,7 @@
     """
     return 1/z
 
+
 def test_coercion(int a, float b, double c, float complex d, double complex e):
     """
     >>> test_coercion(1, 1.5, 2.5, 4+1j, 10j)
@@ -98,36 +135,44 @@
     z = e; print z
     return z + a + b + c + d + e
 
+
 def test_compare(double complex a, double complex b):
     """
     >>> test_compare(3, 3)
-    (True, False)
+    (True, False, False, False, False, True, False)
     >>> test_compare(3j, 3j)
-    (True, False)
+    (True, False, True, True, True, False, False)
     >>> test_compare(3j, 4j)
-    (False, True)
+    (False, True, True, False, True, True, False)
     >>> test_compare(3, 4)
-    (False, True)
+    (False, True, False, False, False, True, False)
+    >>> test_compare(2-1j, 4)
+    (False, True, False, False, False, True, True)
     """
-    return a == b, a != b
+    return a == b, a != b, a == 3j, 3j == b, a == Complex3j(), Complex3j() != b, a == C21
+
 
 def test_compare_coerce(double complex a, int b):
     """
     >>> test_compare_coerce(3, 4)
-    (False, True)
+    (False, True, False, False, False, True)
     >>> test_compare_coerce(4+1j, 4)
-    (False, True)
+    (False, True, False, True, False, True)
     >>> test_compare_coerce(4, 4)
-    (True, False)
+    (True, False, False, False, False, True)
+    >>> test_compare_coerce(3j, 4)
+    (False, True, True, False, True, False)
     """
-    return a == b, a != b
+    return a == b, a != b, a == 3j, 4+1j == a, a == Complex3j(), Complex3j() != a
+
 
 def test_literal():
     """
     >>> test_literal()
-    (5j, (1-2.5j))
+    (5j, (1-2.5j), (2-1j))
     """
-    return 5j, 1-2.5j
+    return 5j, 1-2.5j, C21
+
 
 def test_real_imag(double complex z):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/constant_folding.py cython-0.20.1+1~202203241016-9537/tests/run/constant_folding.py
--- cython-0.20.1+1~201611251650-6686/tests/run/constant_folding.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/constant_folding.py	2022-03-24 10:16:46.000000000 +0000
@@ -5,6 +5,17 @@
 import cython
 
 
+import sys
+IS_PY2 = sys.version_info < (3, 0)
+
+
+def print_big_ints(t):
+    s = repr(t)
+    if IS_PY2:
+        s = s.replace('L', '')
+    print(s)
+
+
 @cython.test_fail_if_path_exists(
     "//UnaryMinusNode",
     "//UnaryPlusNode",
@@ -110,6 +121,33 @@
 
 
 @cython.test_fail_if_path_exists(
+    "//MulNode",
+    "//PowNode",
+)
+def binop_mul_pow():
+    """
+    >>> print_big_ints(binop_mul_pow())
+    (800, 12193263111263526900, 248832, 12467572902176589255564000298710470656)
+    """
+    mul_int = 20 * 40
+    mul_large_int = 1234567890 * 9876543210
+    pow_int = 12 ** 5
+    pow_large_int = 1234 ** 12
+    return (mul_int, mul_large_int, pow_int, pow_large_int)
+
+
+def binop_pow_negative():
+    """
+    >>> print_big_ints(binop_pow_negative())
+    (4.018775720164609e-06, 8.020807320287816e-38, 0.1)
+    """
+    pow_int = 12 ** -5
+    pow_large_int = 1234 ** -12
+    pow_expression_int = 10 ** (1-2)
+    return (pow_int, pow_large_int, pow_expression_int)
+
+
+@cython.test_fail_if_path_exists(
     "//SliceIndexNode",
 )
 def slicing2():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/constants.pyx cython-0.20.1+1~202203241016-9537/tests/run/constants.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/constants.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/constants.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,287 @@
+import sys
+IS_PY3 = sys.version_info[0] >= 3
+
+cimport cython
+
+DEF INT_VAL = 1
+
+def _func(a,b,c):
+    return a+b+c
+
+@cython.test_fail_if_path_exists("//AddNode")
+def add():
+    """
+    >>> add() == 1+2+3+4
+    True
+    """
+    return 1+2+3+4
+
+#@cython.test_fail_if_path_exists("//AddNode")
+def add_var(a):
+    """
+    >>> add_var(10) == 1+2+10+3+4
+    True
+    """
+    return 1+2 +a+ 3+4
+
+@cython.test_fail_if_path_exists("//AddNode", "//SubNode")
+def neg():
+    """
+    >>> neg() == -1 -2 - (-3+4)
+    True
+    """
+    return -1 -2 - (-3+4)
+
+@cython.test_fail_if_path_exists("//AddNode", "//MulNode", "//DivNode")
+def long_int_mix():
+    """
+    >>> long_int_mix() == 1 + (2 * 3) // 2
+    True
+    >>> if IS_PY3: type(long_int_mix()) is int  or type(long_int_mix())
+    ... else:      type(long_int_mix()) is long or type(long_int_mix())
+    True
+    """
+    return 1L + (2 * 3L) // 2
+
+@cython.test_fail_if_path_exists("//AddNode", "//MulNode", "//DivNode")
+def char_int_mix():
+    """
+    >>> char_int_mix() == 1 + (ord(' ') * 3) // 2 + ord('A')
+    True
+    """
+    return 1L + (c' ' * 3L) // 2 + c'A'
+
+@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
+def int_cast():
+    """
+    >>> int_cast() == 1 + 2 * 6000
+    True
+    """
+    return (1 + 2 * 6000)
+
+@cython.test_fail_if_path_exists("//MulNode")
+def mul():
+    """
+    >>> mul() == 1*60*1000
+    True
+    """
+    return 1*60*1000
+
+@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
+def arithm():
+    """
+    >>> arithm() == 9*2+3*8//6-10
+    True
+    """
+    return 9*2+3*8//6-10
+
+@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
+def parameters():
+    """
+    >>> parameters() == _func(-1 -2, - (-3+4), 1*2*3)
+    True
+    """
+    return _func(-1 -2, - (-3+4), 1*2*3)
+
+#@cython.test_fail_if_path_exists("//AddNode")
+def lists():
+    """
+    >>> lists() == [1,2,3] + [4,5,6]
+    True
+    """
+    return [1,2,3] + [4,5,6]
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_right_len1():
+    """
+    >>> multiplied_lists_right_len1() == [1] * 5
+    True
+    """
+    return [1] * 5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_right():
+    """
+    >>> multiplied_lists_right() == [1,2,3] * 5
+    True
+    """
+    return [1,2,3] * 5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_left():
+    """
+    >>> multiplied_lists_left() == [1,2,3] * 5
+    True
+    """
+    return 5 * [1,2,3]
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_neg():
+    """
+    >>> multiplied_lists_neg() == [1,2,3] * -5
+    True
+    """
+    return [1,2,3] * -5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_nonconst(x):
+    """
+    >>> multiplied_lists_nonconst(5) == [1,2,3] * 5
+    True
+    >>> multiplied_lists_nonconst(-5) == [1,2,3] * -5
+    True
+    >>> multiplied_lists_nonconst(0) == [1,2,3] * 0
+    True
+
+    >>> try: [1,2,3] * 'abc'
+    ... except TypeError: pass
+    >>> try: multiplied_nonconst_tuple_arg('abc')
+    ... except TypeError: pass
+    >>> try: [1,2,3] * 1.0
+    ... except TypeError: pass
+    >>> try: multiplied_nonconst_tuple_arg(1.0)
+    ... except TypeError: pass
+    """
+    return [1,2,3] * x
+
+@cython.test_assert_path_exists("//MulNode")
+def multiplied_lists_nonconst_left(x):
+    """
+    >>> multiplied_lists_nonconst_left(5) == 5 * [1,2,3]
+    True
+    >>> multiplied_lists_nonconst_left(-5) == -5 * [1,2,3]
+    True
+    >>> multiplied_lists_nonconst_left(0) == 0 * [1,2,3]
+    True
+    """
+    return x * [1,2,3]
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_nonconst_list_const_int(x):
+    """
+    >>> multiplied_nonconst_list_const_int(2)
+    [1, 2, 3, 1, 2, 3]
+    """
+    return [1,x,3] * 2
+
+
+@cython.test_fail_if_path_exists("//MulNode//ListNode")
+def multiplied_lists_nonconst_expression(x):
+    """
+    >>> multiplied_lists_nonconst_expression(5) == [1,2,3] * (5 * 2)
+    True
+    >>> multiplied_lists_nonconst_expression(-5) == [1,2,3] * (-5 * 2)
+    True
+    >>> multiplied_lists_nonconst_expression(0) == [1,2,3] * (0 * 2)
+    True
+    """
+    return [1,2,3] * (x*2)
+
+cdef side_effect(int x):
+    print x
+    return x
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_with_side_effects():
+    """
+    >>> multiplied_lists_with_side_effects() == [1,2,3] * 5
+    1
+    2
+    3
+    True
+    """
+    return [side_effect(1), side_effect(2), side_effect(3)] * 5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_lists_nonconst_with_side_effects(x):
+    """
+    >>> multiplied_lists_nonconst_with_side_effects(5) == [1,2,3] * 5
+    1
+    2
+    3
+    True
+    """
+    return [side_effect(1), side_effect(2), side_effect(3)] * x
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_nonconst_tuple_arg(x):
+    """
+    >>> multiplied_nonconst_tuple_arg(5) == (1,2) * 5
+    True
+    >>> multiplied_nonconst_tuple_arg(-5) == (1,2) * -5
+    True
+    >>> multiplied_nonconst_tuple_arg(0) == (1,2) * 0
+    True
+
+    >>> try: (1,2) * 'abc'
+    ... except TypeError: pass
+    >>> try: multiplied_nonconst_tuple_arg('abc')
+    ... except TypeError: pass
+    >>> try: (1,2) * 1.0
+    ... except TypeError: pass
+    >>> try: multiplied_nonconst_tuple_arg(1.0)
+    ... except TypeError: pass
+    """
+    return (1,2) * x
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_nonconst_tuple_int_arg(int x):
+    """
+    >>> multiplied_nonconst_tuple_int_arg(5) == (1,2) * 5
+    True
+    """
+    return (1,2) * x
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_nonconst_tuple(x):
+    """
+    >>> multiplied_nonconst_tuple(5) == (1,2) * (5+1)
+    True
+    """
+    return (1,2) * (x + 1)
+
+MULT = 5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_global_nonconst_tuple():
+    """
+    >>> multiplied_global_nonconst_tuple() == (1,2,3) * 5
+    1
+    2
+    3
+    True
+    """
+    return (side_effect(1), side_effect(2), side_effect(3)) * MULT
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_const_tuple():
+    """
+    >>> multiplied_const_tuple() == (1,2) * 5
+    True
+    """
+    return (1,2) * 5
+
+@cython.test_fail_if_path_exists("//MulNode")
+def multiplied_const_tuple_len1():
+    """
+    >>> multiplied_const_tuple_len1() == (1,) * 5
+    True
+    """
+    return (1,) * 5
+
+@cython.test_fail_if_path_exists("//PrimaryCmpNode")
+def compile_time_DEF():
+    """
+    >>> compile_time_DEF()
+    (1, False, True, True, False)
+    """
+    return INT_VAL, INT_VAL == 0, INT_VAL != 0, INT_VAL == 1, INT_VAL != 1
+
+@cython.test_fail_if_path_exists("//PrimaryCmpNode")
+def cascaded_compare():
+    """
+    >>> cascaded_compare()
+    True
+    """
+    return 1 < 2 < 3 < 4
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/consts.pyx cython-0.20.1+1~202203241016-9537/tests/run/consts.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/consts.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/consts.pyx	1970-01-01 00:00:00.000000000 +0000
@@ -1,278 +0,0 @@
-import sys
-IS_PY3 = sys.version_info[0] >= 3
-
-cimport cython
-
-DEF INT_VAL = 1
-
-def _func(a,b,c):
-    return a+b+c
-
-@cython.test_fail_if_path_exists("//AddNode")
-def add():
-    """
-    >>> add() == 1+2+3+4
-    True
-    """
-    return 1+2+3+4
-
-#@cython.test_fail_if_path_exists("//AddNode")
-def add_var(a):
-    """
-    >>> add_var(10) == 1+2+10+3+4
-    True
-    """
-    return 1+2 +a+ 3+4
-
-@cython.test_fail_if_path_exists("//AddNode", "//SubNode")
-def neg():
-    """
-    >>> neg() == -1 -2 - (-3+4)
-    True
-    """
-    return -1 -2 - (-3+4)
-
-@cython.test_fail_if_path_exists("//AddNode", "//MulNode", "//DivNode")
-def long_int_mix():
-    """
-    >>> long_int_mix() == 1 + (2 * 3) // 2
-    True
-    >>> if IS_PY3: type(long_int_mix()) is int  or type(long_int_mix())
-    ... else:      type(long_int_mix()) is long or type(long_int_mix())
-    True
-    """
-    return 1L + (2 * 3L) // 2
-
-@cython.test_fail_if_path_exists("//AddNode", "//MulNode", "//DivNode")
-def char_int_mix():
-    """
-    >>> char_int_mix() == 1 + (ord(' ') * 3) // 2 + ord('A')
-    True
-    """
-    return 1L + (c' ' * 3L) // 2 + c'A'
-
-@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
-def int_cast():
-    """
-    >>> int_cast() == 1 + 2 * 6000
-    True
-    """
-    return (1 + 2 * 6000)
-
-@cython.test_fail_if_path_exists("//MulNode")
-def mul():
-    """
-    >>> mul() == 1*60*1000
-    True
-    """
-    return 1*60*1000
-
-@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
-def arithm():
-    """
-    >>> arithm() == 9*2+3*8//6-10
-    True
-    """
-    return 9*2+3*8//6-10
-
-@cython.test_fail_if_path_exists("//AddNode", "//MulNode")
-def parameters():
-    """
-    >>> parameters() == _func(-1 -2, - (-3+4), 1*2*3)
-    True
-    """
-    return _func(-1 -2, - (-3+4), 1*2*3)
-
-#@cython.test_fail_if_path_exists("//AddNode")
-def lists():
-    """
-    >>> lists() == [1,2,3] + [4,5,6]
-    True
-    """
-    return [1,2,3] + [4,5,6]
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_right_len1():
-    """
-    >>> multiplied_lists_right_len1() == [1] * 5
-    True
-    """
-    return [1] * 5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_right():
-    """
-    >>> multiplied_lists_right() == [1,2,3] * 5
-    True
-    """
-    return [1,2,3] * 5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_left():
-    """
-    >>> multiplied_lists_left() == [1,2,3] * 5
-    True
-    """
-    return 5 * [1,2,3]
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_neg():
-    """
-    >>> multiplied_lists_neg() == [1,2,3] * -5
-    True
-    """
-    return [1,2,3] * -5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_nonconst(x):
-    """
-    >>> multiplied_lists_nonconst(5) == [1,2,3] * 5
-    True
-    >>> multiplied_lists_nonconst(-5) == [1,2,3] * -5
-    True
-    >>> multiplied_lists_nonconst(0) == [1,2,3] * 0
-    True
-
-    >>> try: [1,2,3] * 'abc'
-    ... except TypeError: pass
-    >>> try: multiplied_nonconst_tuple_arg('abc')
-    ... except TypeError: pass
-    >>> try: [1,2,3] * 1.0
-    ... except TypeError: pass
-    >>> try: multiplied_nonconst_tuple_arg(1.0)
-    ... except TypeError: pass
-    """
-    return [1,2,3] * x
-
-@cython.test_assert_path_exists("//MulNode")
-def multiplied_lists_nonconst_left(x):
-    """
-    >>> multiplied_lists_nonconst_left(5) == 5 * [1,2,3]
-    True
-    >>> multiplied_lists_nonconst_left(-5) == -5 * [1,2,3]
-    True
-    >>> multiplied_lists_nonconst_left(0) == 0 * [1,2,3]
-    True
-    """
-    return x * [1,2,3]
-
-@cython.test_fail_if_path_exists("//MulNode//ListNode")
-@cython.test_assert_path_exists("//MulNode")
-def multiplied_lists_nonconst_expression(x):
-    """
-    >>> multiplied_lists_nonconst_expression(5) == [1,2,3] * (5 * 2)
-    True
-    >>> multiplied_lists_nonconst_expression(-5) == [1,2,3] * (-5 * 2)
-    True
-    >>> multiplied_lists_nonconst_expression(0) == [1,2,3] * (0 * 2)
-    True
-    """
-    return [1,2,3] * (x*2)
-
-cdef side_effect(int x):
-    print x
-    return x
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_with_side_effects():
-    """
-    >>> multiplied_lists_with_side_effects() == [1,2,3] * 5
-    1
-    2
-    3
-    True
-    """
-    return [side_effect(1), side_effect(2), side_effect(3)] * 5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_lists_nonconst_with_side_effects(x):
-    """
-    >>> multiplied_lists_nonconst_with_side_effects(5) == [1,2,3] * 5
-    1
-    2
-    3
-    True
-    """
-    return [side_effect(1), side_effect(2), side_effect(3)] * x
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_nonconst_tuple_arg(x):
-    """
-    >>> multiplied_nonconst_tuple_arg(5) == (1,2) * 5
-    True
-    >>> multiplied_nonconst_tuple_arg(-5) == (1,2) * -5
-    True
-    >>> multiplied_nonconst_tuple_arg(0) == (1,2) * 0
-    True
-
-    >>> try: (1,2) * 'abc'
-    ... except TypeError: pass
-    >>> try: multiplied_nonconst_tuple_arg('abc')
-    ... except TypeError: pass
-    >>> try: (1,2) * 1.0
-    ... except TypeError: pass
-    >>> try: multiplied_nonconst_tuple_arg(1.0)
-    ... except TypeError: pass
-    """
-    return (1,2) * x
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_nonconst_tuple_int_arg(int x):
-    """
-    >>> multiplied_nonconst_tuple_int_arg(5) == (1,2) * 5
-    True
-    """
-    return (1,2) * x
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_nonconst_tuple(x):
-    """
-    >>> multiplied_nonconst_tuple(5) == (1,2) * (5+1)
-    True
-    """
-    return (1,2) * (x + 1)
-
-MULT = 5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_global_nonconst_tuple():
-    """
-    >>> multiplied_global_nonconst_tuple() == (1,2,3) * 5
-    1
-    2
-    3
-    True
-    """
-    return (side_effect(1), side_effect(2), side_effect(3)) * MULT
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_const_tuple():
-    """
-    >>> multiplied_const_tuple() == (1,2) * 5
-    True
-    """
-    return (1,2) * 5
-
-@cython.test_fail_if_path_exists("//MulNode")
-def multiplied_const_tuple_len1():
-    """
-    >>> multiplied_const_tuple_len1() == (1,) * 5
-    True
-    """
-    return (1,) * 5
-
-@cython.test_fail_if_path_exists("//PrimaryCmpNode")
-def compile_time_DEF():
-    """
-    >>> compile_time_DEF()
-    (1, False, True, True, False)
-    """
-    return INT_VAL, INT_VAL == 0, INT_VAL != 0, INT_VAL == 1, INT_VAL != 1
-
-@cython.test_fail_if_path_exists("//PrimaryCmpNode")
-def cascaded_compare():
-    """
-    >>> cascaded_compare()
-    True
-    """
-    return 1 < 2 < 3 < 4
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/contains_T455.pyx cython-0.20.1+1~202203241016-9537/tests/run/contains_T455.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/contains_T455.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/contains_T455.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 455
+# ticket: t455
 
 def in_sequence(x, seq):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coroutines.py cython-0.20.1+1~202203241016-9537/tests/run/coroutines.py
--- cython-0.20.1+1~201611251650-6686/tests/run/coroutines.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coroutines.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,60 @@
+# cython: language_level=3
+# mode: run
+# tag: pep492, pure3.5, gh1462, async, await
+
+
+async def test_coroutine_frame(awaitable):
+    """
+    >>> class Awaitable(object):
+    ...     def __await__(self):
+    ...         return iter([2])
+
+    >>> coro = test_coroutine_frame(Awaitable())
+    >>> import types
+    >>> isinstance(coro.cr_frame, types.FrameType) or coro.cr_frame
+    True
+    >>> coro.cr_frame is coro.cr_frame  # assert that it's cached
+    True
+    >>> coro.cr_frame.f_code is not None
+    True
+    >>> code_obj = coro.cr_frame.f_code
+    >>> code_obj.co_argcount
+    1
+    >>> code_obj.co_varnames
+    ('awaitable', 'b')
+
+    >>> next(coro.__await__())  # avoid "not awaited" warning
+    2
+    """
+    b = await awaitable
+    return b
+
+
+# gh1462: Using decorators on coroutines.
+
+def pass_through(func):
+    return func
+
+
+@pass_through
+async def test_pass_through():
+    """
+    >>> t = test_pass_through()
+    >>> try: t.send(None)
+    ... except StopIteration as ex:
+    ...     print(ex.args[0] if ex.args else None)
+    ... else: print("NOT STOPPED!")
+    None
+    """
+
+
+@pass_through(pass_through)
+async def test_pass_through_with_args():
+    """
+    >>> t = test_pass_through_with_args()
+    >>> try: t.send(None)
+    ... except StopIteration as ex:
+    ...     print(ex.args[0] if ex.args else None)
+    ... else: print("NOT STOPPED!")
+    None
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coverage_api.srctree cython-0.20.1+1~202203241016-9537/tests/run/coverage_api.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/coverage_api.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coverage_api.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -2,7 +2,7 @@
 # tag: coverage,trace
 
 """
-PYTHON -c 'import shutil; shutil.copy("pkg/coverage_test_pyx.pyx", "pkg/coverage_test_pyx.pxi")'
+PYTHON -c "import shutil; shutil.copy('pkg/coverage_test_pyx.pyx', 'pkg/coverage_test_pyx.pxi')"
 PYTHON setup.py build_ext -i
 PYTHON coverage_test.py
 """
@@ -14,7 +14,8 @@
 
 setup(ext_modules = cythonize([
     'coverage_test_*.py*',
-    'pkg/coverage_test_*.py*'
+    'pkg/coverage_test_*.py*',
+    'Package2/CoverageTest_*.py*'
 ]))
 
 
@@ -68,6 +69,52 @@
     return cfunc1(x) + func1(x, 4) + func2(x)    # 12
 
 
+######## Package2/__init__.py ########
+# Add MixedCase package and filenames to test if the files are found
+
+######## Package2/CoverageTest_py.py ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+def func1(a, b):
+    x = 1               #  5
+    c = func2(a) + b    #  6
+    return x + c        #  7
+
+
+def func2(a):
+    return a * 2        # 11
+
+
+######## Package2/CoverageTest_pyx.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+def func1(int a, int b):
+    cdef int x = 1      #  5
+    c = func2(a) + b    #  6
+    return x + c        #  7
+
+
+def func2(int a):
+    return a * 2        # 11
+
+
+######## coverage_test_include_pyx.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+cdef int x = 5                                   #  4
+
+cdef int cfunc1(int x):                          #  6
+    return x * 3                                 #  7
+
+include "pkg/coverage_test_pyx.pxi"              #  9
+
+def main_func(int x):                            # 11
+    return cfunc1(x) + func1(x, 4) + func2(x)    # 12
+
+
 ######## coverage_test.py ########
 
 import re
@@ -84,8 +131,12 @@
 from pkg import coverage_test_pyx
 import coverage_test_include_pyx
 
+# test the MixedCase Files and packages
+from Package2 import CoverageTest_py
+from Package2 import CoverageTest_pyx
 
-for module in [coverage_test_py, coverage_test_pyx, coverage_test_include_pyx]:
+for module in [coverage_test_py, coverage_test_pyx, coverage_test_include_pyx,
+               CoverageTest_py, CoverageTest_pyx]:
     assert not any(module.__file__.endswith(ext) for ext in '.py .pyc .pyo .pyw .pyx .pxi'.split()), \
         module.__file__
 
@@ -93,10 +144,18 @@
 def source_file_for(module):
     module_name = module.__name__
     path, ext = os.path.splitext(module.__file__)
-    platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[^.]*$', path, re.I)
-    if platform_suffix:
-        path = path[:platform_suffix.start()]
-    return path + '.' + module_name.rsplit('_', 1)[-1]
+    if ext == '.so':
+        # Linux/Unix/Mac extension module
+        platform_suffix = re.search(r'[.](?:cpython|pypy)-[0-9]+[-_a-z0-9]*$', path, re.I)
+        if platform_suffix:
+            path = path[:platform_suffix.start()]
+    elif ext == '.pyd':
+        # Windows extension module
+        platform_suffix = re.search(r'[.]cp[0-9]+-win[_a-z0-9]*$', path, re.I)
+        if platform_suffix:
+            path = path[:platform_suffix.start()]
+    source_filepath = path + '.' + module_name.rsplit('_', 1)[-1]
+    return source_filepath
 
 
 def run_coverage(module):
@@ -137,3 +196,5 @@
     run_coverage(coverage_test_py)
     run_coverage(coverage_test_pyx)
     run_coverage(coverage_test_include_pyx)
+    run_coverage(CoverageTest_py)
+    run_coverage(CoverageTest_pyx)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coverage_cmd_src_layout.srctree cython-0.20.1+1~202203241016-9537/tests/run/coverage_cmd_src_layout.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/coverage_cmd_src_layout.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coverage_cmd_src_layout.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,167 @@
+# mode: run
+# tag: coverage,trace
+
+"""
+PYTHON setup.py build_ext -i
+PYTHON -m coverage run --source=src coverage_test.py
+PYTHON collect_coverage.py
+"""
+
+######## setup.py ########
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize([
+    'src/trivial_module.pyx',
+]))
+
+
+######## .coveragerc ########
+[run]
+plugins = Cython.Coverage
+
+
+######## src/trivial_module.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+def func1(int a, int b):
+    cdef int x = 1      #  5
+    c = func2(a) + b    #  6
+    return x + c        #  7
+
+
+def func2(int a):
+    return a * 2        # 11
+
+
+######## coverage_test.py ########
+
+import os.path
+import trivial_module
+
+
+assert not any(
+    trivial_module.__file__.endswith(ext)
+    for ext in '.py .pyc .pyo .pyw .pyx .pxi'.split()
+), module.__file__
+
+
+def run_coverage(module):
+    assert module.func1(1, 2) == (1 * 2) + 2 + 1
+    assert module.func2(2) == 2 * 2
+
+
+if __name__ == '__main__':
+    run_coverage(trivial_module)
+
+
+######## collect_coverage.py ########
+
+import re
+import sys
+import os
+import os.path
+import subprocess
+from glob import iglob
+
+
+def run_coverage_command(*command):
+    env = dict(os.environ, LANG='', LC_ALL='C')
+    process = subprocess.Popen(
+        [sys.executable, '-m', 'coverage'] + list(command),
+        stdout=subprocess.PIPE, env=env)
+    stdout, _ = process.communicate()
+    return stdout
+
+
+def run_report():
+    stdout = run_coverage_command('report', '--show-missing')
+    stdout = stdout.decode('iso8859-1')  # 'safe' decoding
+    lines = stdout.splitlines()
+    print(stdout)
+
+    module_path = 'trivial_module.pyx'
+    assert any(module_path in line for line in lines), (
+        "'%s' not found in coverage report:\n\n%s" % (module_path, stdout))
+
+    files = {}
+    line_iter = iter(lines)
+    for line in line_iter:
+        if line.startswith('---'):
+            break
+    extend = [''] * 2
+    for line in line_iter:
+        if not line or line.startswith('---'):
+            continue
+        name, statements, missed, covered, _missing = (line.split(None, 4) + extend)[:5]
+        missing = []
+        for start, end in re.findall('([0-9]+)(?:-([0-9]+))?', _missing):
+            if end:
+                missing.extend(range(int(start), int(end)+1))
+            else:
+                missing.append(int(start))
+        files[os.path.basename(name)] = (statements, missed, covered, missing)
+
+    assert  5 not in files[module_path][-1], files[module_path]
+    assert  6 not in files[module_path][-1], files[module_path]
+    assert  7 not in files[module_path][-1], files[module_path]
+    assert 11 not in files[module_path][-1], files[module_path]
+
+
+def run_xml_report():
+    stdout = run_coverage_command('xml', '-o', '-')
+    print(stdout)
+
+    import xml.etree.ElementTree as etree
+    data = etree.fromstring(stdout)
+
+    files = {}
+    for module in data.iterfind('.//class'):
+        files[module.get('filename').replace('\\', '/')] = dict(
+            (int(line.get('number')), int(line.get('hits')))
+            for line in module.findall('lines/line')
+        )
+
+    module_path = 'src/trivial_module.pyx'
+
+    assert files[module_path][5] > 0, files[module_path]
+    assert files[module_path][6] > 0, files[module_path]
+    assert files[module_path][7] > 0, files[module_path]
+    assert files[module_path][11] > 0, files[module_path]
+
+
+def run_html_report():
+    from collections import defaultdict
+
+    stdout = run_coverage_command('html', '-d', 'html')
+    # coverage 6.1+ changed the order of the attributes => need to parse them separately
+    _parse_id = re.compile(r'id=["\'][^0-9"\']*(?P[0-9]+)[^0-9"\']*["\']').search
+    _parse_state = re.compile(r'class=["\'][^"\']*(?Pmis|run|exc)[^"\']*["\']').search
+
+    files = {}
+    for file_path in iglob('html/*.html'):
+        with open(file_path) as f:
+            page = f.read()
+        report = defaultdict(set)
+        for line in re.split(r'id=["\']source["\']', page)[-1].splitlines():
+            lineno = _parse_id(line)
+            state = _parse_state(line)
+            if not lineno or not state:
+                continue
+            report[state.group('state')].add(int(lineno.group('id')))
+        files[file_path] = (report['run'], report['mis'])
+
+    executed, missing = [data for path, data in files.items() if 'trivial_module' in path][0]
+    assert executed
+    assert 5 in executed, executed
+    assert 6 in executed, executed
+    assert 7 in executed, executed
+    assert 11 in executed, executed
+
+
+if __name__ == '__main__':
+    run_report()
+    run_xml_report()
+    run_html_report()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coverage_cmd.srctree cython-0.20.1+1~202203241016-9537/tests/run/coverage_cmd.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/coverage_cmd.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coverage_cmd.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -2,7 +2,7 @@
 # tag: coverage,trace
 
 """
-PYTHON -c 'import shutil; shutil.copy("pkg/coverage_test_pyx.pyx", "pkg/coverage_test_pyx.pxi")'
+PYTHON -c "import shutil; shutil.copy('pkg/coverage_test_pyx.pyx', 'pkg/coverage_test_pyx.pxi')"
 PYTHON setup.py build_ext -i
 PYTHON -m coverage run coverage_test.py
 PYTHON collect_coverage.py
@@ -29,29 +29,59 @@
 ######## pkg/coverage_test_py.py ########
 # cython: linetrace=True
 # distutils: define_macros=CYTHON_TRACE=1
+import cython
+
 
 def func1(a, b):
-    x = 1               #  5
-    c = func2(a) + b    #  6
-    return x + c        #  7
+    x = 1               #  7
+    c = func2(a) + b    #  8
+    return x + c        #  9
 
 
 def func2(a):
-    return a * 2        # 11
+    return a * 2        # 13
+
+
+def func3(a):
+    x = 1               # 17
+    a *= 2              #     # pragma: no cover
+    a += x              #
+    return a * 42       # 20  # pragma: no cover
+
+
+@cython.cclass
+class A:
+    def meth(self):
+        return 1        # 26
 
 
 ######## pkg/coverage_test_pyx.pyx ########
 # cython: linetrace=True
 # distutils: define_macros=CYTHON_TRACE=1
 
+
+
 def func1(int a, int b):
-    cdef int x = 1      #  5
-    c = func2(a) + b    #  6
-    return x + c        #  7
+    cdef int x = 1      #  7
+    c = func2(a) + b    #  8
+    return x + c        #  9
 
 
 def func2(int a):
-    return a * 2        # 11
+    return a * 2        # 13
+
+
+def func3(int a):
+    cdef int x = 1      # 17
+    a *= 2              #     # pragma: no cover
+    a += x              #
+    return a * 42       # 20  # pragma: no cover
+
+
+
+cdef class A:
+    def meth(self):
+        return 1        # 26
 
 
 ######## coverage_test_include_pyx.pyx ########
@@ -96,6 +126,9 @@
     assert module.func2(2) == 2 * 2
     if '_include_' in module_name:
         assert module.main_func(2) == (2 * 3) + ((2 * 2) + 4 + 1) + (2 * 2)
+    assert module.A().meth() == 1
+
+
 
 
 if __name__ == '__main__':
@@ -152,8 +185,9 @@
                 missing.append(int(start))
         files[os.path.basename(name)] = (statements, missed, covered, missing)
 
-    assert  7 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx']
-    assert 12 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx']
+    report = files['coverage_test_pyx.pyx']
+    assert  7 not in report[-1], report
+    assert 12 not in report[-1], report
 
 
 def run_xml_report():
@@ -170,35 +204,80 @@
             for line in module.findall('lines/line')
         )
 
-    assert files['pkg/coverage_test_pyx.pyx'][5] > 0, files['pkg/coverage_test_pyx.pyx']
-    assert files['pkg/coverage_test_pyx.pyx'][6] > 0, files['pkg/coverage_test_pyx.pyx']
-    assert files['pkg/coverage_test_pyx.pyx'][7] > 0, files['pkg/coverage_test_pyx.pyx']
+    report = files['pkg/coverage_test_pyx.pyx']
+    assert report[7] > 0, report
+    assert report[8] > 0, report
+    assert report[9] > 0, report
+    assert report[26] > 0, report
+
+
+def run_json_report():
+    import coverage
+    if coverage.version_info < (5, 0):
+        # JSON output comes in coverage 5.0
+        return
+
+    stdout = run_coverage_command('json', '-o', '-')
+
+    import json
+    files = json.loads(stdout.decode("ascii"))['files']
+
+    for filename in [
+        'pkg/coverage_test_py.py',
+        'pkg/coverage_test_pyx.pyx',
+    ]:
+        report = files[filename.replace('/', os.sep)]
+        summary = report['summary']
+        assert summary['missing_lines'] == 2, summary
+        assert summary['excluded_lines'] == 2, summary
+        assert report['missing_lines'] == [17, 19], report
+        assert report['excluded_lines'] == [18, 20], report
+
+        assert not frozenset(
+            report['missing_lines'] + report['excluded_lines']
+        ).intersection(report['executed_lines'])
 
 
 def run_html_report():
+    from collections import defaultdict
+
     stdout = run_coverage_command('html', '-d', 'html')
-    _parse_lines = re.compile(
-        r']* id=["\'][^0-9"\']*(?P[0-9]+)[^0-9"\']*["\'][^>]*'
-        r' class=["\'][^"\']*(?Pmis|run)[^"\']*["\']').findall
+    # coverage 6.1+ changed the order of the attributes => need to parse them separately
+    _parse_id = re.compile(r'id=["\'][^0-9"\']*(?P[0-9]+)[^0-9"\']*["\']').search
+    _parse_state = re.compile(r'class=["\'][^"\']*(?Pmis|run|exc)[^"\']*["\']').search
 
     files = {}
     for file_path in iglob('html/*.html'):
         with open(file_path) as f:
             page = f.read()
-        executed = set()
-        missing = set()
-        for line, has_run in _parse_lines(page):
-            (executed if has_run == 'run' else missing).add(int(line))
-        files[file_path] = (executed, missing)
-
-    executed, missing = [data for path, data in files.items() if 'coverage_test_pyx' in path][0]
-    assert executed
-    assert 5 in executed, executed
-    assert 6 in executed, executed
-    assert 7 in executed, executed
+        report = defaultdict(set)
+        for line in re.split(r'id=["\']source["\']', page)[-1].splitlines():
+            lineno = _parse_id(line)
+            state = _parse_state(line)
+            if not lineno or not state:
+                continue
+            report[state.group('state')].add(int(lineno.group('id')))
+        files[file_path] = report
+
+    for filename, report in files.items():
+        if "coverage_test_pyx" not in filename:
+            continue
+        executed = report["run"]
+        missing = report["mis"]
+        excluded = report["exc"]
+        assert executed, (filename, report)
+        assert 7 in executed, executed
+        assert 8 in executed, executed
+        assert 9 in executed, executed
+        assert 26 in executed, executed
+        assert 17 in missing, missing
+        assert 18 in excluded, excluded
+        assert 19 in missing, missing
+        assert 20 in excluded, excluded
 
 
 if __name__ == '__main__':
     run_report()
     run_xml_report()
+    run_json_report()
     run_html_report()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coverage_installed_pkg.srctree cython-0.20.1+1~202203241016-9537/tests/run/coverage_installed_pkg.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/coverage_installed_pkg.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coverage_installed_pkg.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,71 @@
+# mode: run
+# tag: coverage,trace
+
+"""
+PYTHON setup.py build_ext -i
+PYTHON -c "import shutil; shutil.move('ext_src/ext_pkg', 'ext_pkg')"
+PYTHON -m coverage run coverage_test.py
+PYTHON -m coverage report
+"""
+
+######## setup.py ########
+from distutils.core import setup, Extension
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize([
+    'pkg/*.pyx',
+]))
+
+setup(
+    name='ext_pkg',
+    package_dir={'': 'ext_src'},
+    ext_modules = cythonize([
+        Extension('ext_pkg._mul', ['ext_src/ext_pkg/mul.py'])
+    ]),
+)
+
+
+######## .coveragerc ########
+[run]
+plugins = Cython.Coverage
+
+
+######## pkg/__init__.py ########
+from .test_ext_import import test_add
+
+
+######## pkg/test_ext_import.pyx ########
+# cython: linetrace=True
+# distutils: define_macros=CYTHON_TRACE=1
+
+import ext_pkg
+
+
+cpdef test_add(int a, int b):
+    return a + ext_pkg.test_mul(b, 2)
+
+
+######## ext_src/ext_pkg/__init__.py ########
+from .mul import test_mul
+
+
+######## ext_src/ext_pkg/mul.py ########
+from __future__ import absolute_import
+
+
+def test_mul(a, b):
+     return a * b
+
+
+try:
+    from ._mul import *
+except ImportError:
+    pass
+
+
+######## coverage_test.py ########
+
+from pkg import test_add
+
+
+assert 5 == test_add(1, 2)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/coverage_nogil.srctree cython-0.20.1+1~202203241016-9537/tests/run/coverage_nogil.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/coverage_nogil.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/coverage_nogil.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: coverage,trace,nogil
+# tag: coverage,trace,nogil,fastgil
 
 """
 PYTHON setup.py build_ext -i
@@ -21,23 +21,35 @@
 plugins = Cython.Coverage
 
 
-######## coverage_test_nogil.pyx ########
-# cython: linetrace=True
+######## coverage_test_nogil_fastgil.pyx ########
+# cython: linetrace=True,fast_gil=True
 # distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1
+include "_coverage_test_nogil.pxi"
 
-cdef int func1(int a, int b) nogil:
-    cdef int x                   #  5
-    with gil:                    #  6
-        x = 1                    #  7
-    cdef int c = func2(a) + b    #  8
-    return x + c                 #  9
 
-
-cdef int func2(int a) with gil:
-    return a * 2                 # 13
+######## coverage_test_nogil_nofastgil.pyx ########
+# cython: linetrace=True,fast_gil=False
+# distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1
+include "_coverage_test_nogil.pxi"
 
 
-def call(int a, int b):
+######## _coverage_test_nogil.pxi ########
+#  1
+#  2
+#  3
+cdef int func1(int a, int b) nogil:  #  4
+    cdef int x                       #  5
+    with gil:                        #  6
+        x = 1                        #  7
+    cdef int c = func2(a) + b        #  8
+    return x + c                     #  9
+# 10
+# 11
+cdef int func2(int a) with gil:  # 12
+    return a * 2                 # 13
+# 14
+# 15
+def call(int a, int b):          # 16
     a, b = b, a                  # 17
     with nogil:                  # 18
         result = func1(b, a)     # 19
@@ -56,22 +68,20 @@
 from coverage import coverage
 
 
-import coverage_test_nogil
-
-assert not any(coverage_test_nogil.__file__.endswith(ext)
-               for ext in '.py .pyc .pyo .pyw .pyx .pxi'.split()), \
-    coverage_test_nogil.__file__
-
+def run_coverage(module_name):
+    print("Testing module %s" % module_name)
+    cov = coverage()
+    cov.start()
 
-def run_coverage(module):
+    module = __import__(module_name)
     module_name = module.__name__
     module_path = module_name + '.pyx'
-
-    cov = coverage()
-    cov.start()
+    assert not any(module.__file__.endswith(ext)
+                   for ext in '.py .pyc .pyo .pyw .pyx .pxi'.split()), \
+        module.__file__
     assert module.call(1, 2) == (1 * 2) + 2 + 1
-    cov.stop()
 
+    cov.stop()
     out = StringIO()
     cov.report(file=out)
     #cov.report([module], file=out)
@@ -79,15 +89,17 @@
     assert any(module_path in line for line in lines), \
         "'%s' not found in coverage report:\n\n%s" % (module_path, out.getvalue())
 
-    mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_path))
-    assert module_path in mod_file
+    module_pxi = "_coverage_test_nogil.pxi"
+    mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_pxi))
+    assert module_pxi in mod_file
 
     executed = set(exec_lines) - set(missing_lines)
-    # check that everything that runs with the gil owned was executed
+    # check that everything that runs with the gil owned was executed (missing due to pxi: 4, 12, 16)
     assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines)
     # check that everything that runs in nogil sections was executed
     assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines)
 
 
 if __name__ == '__main__':
-    run_coverage(coverage_test_nogil)
+    for module_name in ["coverage_test_nogil_fastgil", "coverage_test_nogil_nofastgil"]:
+        run_coverage(module_name)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_enums.pxd cython-0.20.1+1~202203241016-9537/tests/run/cpdef_enums.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_enums.pxd	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_enums.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -11,5 +11,19 @@
     RANK_1 = 37
     RANK_2 = 389
 
+cpdef enum cpdefPxdDocEnum:
+    """Home is where...
+    """
+    RANK_6 = 159
+
+cpdef enum cpdefPxdDocLineEnum:
+    """Home is where..."""
+    RANK_7 = 889
+
 cdef enum PxdSecretEnum:
-    RANK_3 = 5077
+    RANK_8 = 5077
+
+cdef enum cdefPxdDocEnum:
+    """the heart is.
+    """
+    RANK_9 = 2458
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_enums.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_enums.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_enums.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_enums.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,6 @@
 """
+>>> import sys
+
 >>> ONE, TEN, HUNDRED
 (1, 10, 100)
 >>> THOUSAND        # doctest: +ELLIPSIS
@@ -11,6 +13,8 @@
 True
 >>> FIVE == 5 or FIVE
 True
+>>> ELEVEN == 11 or ELEVEN
+True
 >>> SEVEN           # doctest: +ELLIPSIS
 Traceback (most recent call last):
 NameError: ...name 'SEVEN' is not defined
@@ -29,25 +33,31 @@
 True
 >>> RANK_2 == 389 or RANK_2
 True
+>>> RANK_6 == 159 or RANK_6
+True
+>>> RANK_7 == 889 or RANK_7
+True
 >>> RANK_3         # doctest: +ELLIPSIS
 Traceback (most recent call last):
 NameError: ...name 'RANK_3' is not defined
 
->>> set(PyxEnum) == set([TWO, THREE, FIVE])
+>>> set(PyxEnum) == {TWO, THREE, FIVE}
 True
->>> str(PyxEnum.TWO)
-'PyxEnum.TWO'
+>>> str(PyxEnum.TWO).split(".")[-1]  if sys.version_info < (3,11) else  "TWO" # Py3.10/11 changed the output here
+'TWO'
+>>> str(PyxEnum.TWO)  if sys.version_info >= (3,11) else  "2" # Py3.10/11 changed the output here
+'2'
 >>> PyxEnum.TWO + PyxEnum.THREE == PyxEnum.FIVE
 True
 >>> PyxEnum(2) is PyxEnum["TWO"] is PyxEnum.TWO
 True
 
->>> IntEnum  # not leaking into module namespace
+# not leaking into module namespace
+>>> IntEnum        # doctest: +ELLIPSIS
 Traceback (most recent call last):
-NameError: name 'IntEnum' is not defined
+NameError: ...name 'IntEnum' is not defined
 """
 
-
 cdef extern from *:
     cpdef enum: # ExternPyx
         ONE "1"
@@ -62,21 +72,30 @@
     THREE = 3
     FIVE = 5
 
+cpdef enum cpdefPyxDocEnum:
+    """Home is where...
+    """
+    ELEVEN = 11
+
+cpdef enum cpdefPyxDocLineEnum:
+    """Home is where..."""
+    FOURTEEN = 14
+
 cdef enum SecretPyxEnum:
     SEVEN = 7
 
+cdef enum cdefPyxDocEnum:
+    """the heart is.
+    """
+    FIVE_AND_SEVEN = 5077
+
+
 def test_as_variable_from_cython():
     """
     >>> test_as_variable_from_cython()
     """
-    import sys
-    if sys.version_info >= (2, 7):
-        assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum)
-        assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum)
-    else:
-        # No OrderedDict.
-        assert set(PyxEnum) == {TWO, THREE, FIVE}, list(PyxEnum)
-        assert set(PxdEnum) == {RANK_0, RANK_1, RANK_2}, list(PxdEnum)
+    assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum)
+    assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum)
 
 cdef int verify_pure_c() nogil:
     cdef int x = TWO
@@ -86,3 +105,29 @@
 
 # Use it to suppress warning.
 verify_pure_c()
+
+def verify_resolution_GH1533():
+    """
+    >>> verify_resolution_GH1533()
+    3
+    """
+    THREE = 100
+    return int(PyxEnum.THREE)
+
+
+def check_docs():
+    """
+    >>> PxdEnum.__doc__ not in ("Home is where...\\n    ", "Home is where...")
+    True
+    >>> PyxEnum.__doc__ not in ("Home is where...\\n    ", "Home is where...")
+    True
+    >>> cpdefPyxDocEnum.__doc__ == "Home is where...\\n    "
+    True
+    >>> cpdefPxdDocEnum.__doc__ == "Home is where...\\n    "
+    True
+    >>> cpdefPyxDocLineEnum.__doc__
+    'Home is where...'
+    >>> cpdefPxdDocLineEnum.__doc__
+    'Home is where...'
+    """
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_extern_func.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_extern_func.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_extern_func.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_extern_func.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,6 @@
 # cython: c_string_type=str
 # cython: c_string_encoding=ascii
+# distutils: extra_compile_args=-fpermissive
 
 __doc__ = """
 >>> sqrt(1)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_method_override.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_method_override.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_method_override.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_method_override.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,139 @@
+# mode: run
+# tag: cpdef
+# ticket: 1771
+
+def _call_method(cls):
+    obj = cls()
+    obj.callmeth()
+    obj = cls()
+    obj.callmeth()
+    obj.callmeth()
+    obj = cls()
+    obj.callmeth()
+    obj.callmeth()
+    obj.callmeth()
+
+
+cdef class BaseType:
+    """
+    >>> BaseType().callmeth()
+    BaseType.meth
+    >>> obj = BaseType()
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> _call_method(BaseType)
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    """
+    cpdef callmeth(self):
+        return self.callmeth2()
+    cpdef callmeth2(self):
+        # not overridden by subclasses
+        return self.meth()
+    cpdef meth(self):
+        # overridden by subclasses
+        print("BaseType.meth")
+
+
+class NonOverride(BaseType):
+    """
+    >>> NonOverride().callmeth()
+    BaseType.meth
+    >>> obj = NonOverride()
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> _call_method(NonOverride)
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    BaseType.meth
+    """
+
+
+class PyClass(BaseType):
+    """
+    >>> PyClass().callmeth()
+    PyClass.meth
+    >>> obj = PyClass()
+    >>> obj.callmeth()
+    PyClass.meth
+    >>> obj.callmeth()
+    PyClass.meth
+    >>> obj.callmeth()
+    PyClass.meth
+    >>> _call_method(PyClass)
+    PyClass.meth
+    PyClass.meth
+    PyClass.meth
+    PyClass.meth
+    PyClass.meth
+    PyClass.meth
+    """
+    def meth(self):
+        print("PyClass.meth")
+
+
+class PySlotsClass(BaseType):
+    """
+    >>> PySlotsClass().callmeth()
+    PySlotsClass.meth
+    >>> obj = PySlotsClass()
+    >>> obj.callmeth()
+    PySlotsClass.meth
+    >>> obj.callmeth()
+    PySlotsClass.meth
+    >>> obj.callmeth()
+    PySlotsClass.meth
+    >>> _call_method(PySlotsClass)
+    PySlotsClass.meth
+    PySlotsClass.meth
+    PySlotsClass.meth
+    PySlotsClass.meth
+    PySlotsClass.meth
+    PySlotsClass.meth
+    """
+    __slots__ = []
+
+    def meth(self):
+        print("PySlotsClass.meth")
+
+
+class DynamicOverride(BaseType):
+    """
+    >>> DynamicOverride().callmeth()
+    meth1
+    >>> obj = DynamicOverride()
+    >>> obj.callmeth()
+    meth1
+    >>> obj.callmeth()
+    meth2
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> obj.callmeth()
+    BaseType.meth
+    >>> _call_method(DynamicOverride)
+    meth1
+    meth1
+    meth2
+    meth1
+    meth2
+    BaseType.meth
+    """
+    def __init__(self):
+        self.meth = self.meth1
+    def meth1(self):
+        self.meth = self.meth2
+        print("meth1")
+    def meth2(self):
+        del self.meth
+        print("meth2")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_method_override_recursion.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_method_override_recursion.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_method_override_recursion.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_method_override_recursion.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,66 @@
+# mode: run
+# tag: cpdef
+
+# This also makes a nice benchmark for the cpdef method call dispatching code.
+
+cdef class Ext:
+    """
+    >>> x = Ext()
+    >>> x.rec(10)
+    0
+    """
+    cpdef rec(self, int i):
+        return 0 if i < 0 else self.rec(i-1)
+
+
+class Py(Ext):
+    """
+    >>> p = Py()
+    >>> p.rec(10)
+    0
+    """
+    pass
+
+
+class Slots(Ext):
+    """
+    >>> s = Slots()
+    >>> s.rec(10)
+    0
+    """
+    __slots__ = ()
+
+
+class PyOverride(Ext):
+    """
+    >>> p = PyOverride()
+    >>> p.rec(10)
+    10
+    5
+    >>> p.rec(12)
+    12
+    11
+    10
+    5
+    """
+    def rec(self, i):
+        print(i)
+        return Ext.rec(self, i) if i > 10 else 5
+
+
+class SlotsOverride(Ext):
+    """
+    >>> s = SlotsOverride()
+    >>> s.rec(10)
+    10
+    6
+    >>> s.rec(12)
+    12
+    11
+    10
+    6
+    """
+    __slots__ = ()
+    def rec(self, i):
+        print(i)
+        return Ext.rec(self, i) if i > 10 else 6
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_nogil.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_nogil.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_nogil.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_nogil.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,19 @@
+# cython: binding=True
+# mode: run
+# tag: cyfunction
+
+cpdef int simple() nogil:
+    """
+    >>> simple()
+    1
+    """
+    return 1
+
+
+cpdef int call_nogil():
+    """
+    >>> call_nogil()
+    1
+    """
+    with nogil:
+        return simple()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_pickle.srctree cython-0.20.1+1~202203241016-9537/tests/run/cpdef_pickle.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_pickle.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_pickle.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,70 @@
+# mode: run
+# tag: pickle
+
+PYTHON main.py build_ext -i
+
+######################### lib/__init__.py #########################
+
+######################### lib/cy.pyx #########################
+# cython: binding=True
+
+cdef class WithoutC:
+    def hello(self):
+        return "Hello, World"
+
+cdef class WithCPDef:
+    cpdef str hello(self):
+        return "Hello, World"
+
+cdef class WithCDefWrapper:
+    def hello(self):
+        return _helloC(self)
+
+cpdef _helloC(object caller):
+    return "Hello, World"
+
+
+######################### lib/cy.pxd #########################
+# cython:language_level=3
+
+cdef class WithoutCPDef:
+    pass
+
+cdef class WithCPDef:
+    cpdef str hello(self)
+
+cdef class WithCDefWrapper:
+    pass
+
+cpdef _helloC(object caller)
+
+
+######################### main.py #########################
+#!/usr/bin/env python3
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize(["lib/*.pyx"]),
+)
+
+import pickle as pkl
+import os
+from lib.cy import WithoutC, WithCPDef, WithCDefWrapper
+
+def tryThis(obj):
+    print("Pickling %s ..." % obj.__class__.__name__)
+    try:
+        with open("test.pkl", "wb") as fid:
+            pkl.dump(obj, fid)
+        print("\t... OK")
+    except Exception as e:
+        print("\t... KO: %s" % str(e))
+
+try:
+    for t in WithoutC(), WithCPDef(), WithCDefWrapper():
+        tryThis(t)
+finally:
+    if os.path.exists("test.pkl"):
+        os.remove("test.pkl")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_scoped_enums_import.srctree cython-0.20.1+1~202203241016-9537/tests/run/cpdef_scoped_enums_import.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_scoped_enums_import.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_scoped_enums_import.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,71 @@
+# mode: run
+# tag: cpp, cpp11
+
+"""
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+"""
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+setup(ext_modules=cythonize("*.pyx", language='c++'))
+
+setup(
+  ext_modules = cythonize([
+              "cheese.pyx",
+              "import_scoped_enum_test.pyx",
+              "dotted_import_scoped_enum_test.pyx"
+              ])
+)
+
+######## cheese.pxd ########
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
+
+
+cdef extern from * namespace "Namespace":
+    """
+    namespace Namespace {
+        enum class Cheese {
+            cheddar = 1,
+            camembert = 2
+        };
+    }
+    """
+    cpdef enum class Cheese:
+        cheddar
+        camembert
+
+######## cheese.pyx ########
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
+
+pass
+
+######## import_scoped_enum_test.pyx ########
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
+
+from cheese import Cheese
+from cheese cimport Cheese
+
+cdef Cheese c = Cheese.cheddar
+assert list(Cheese) == [1, 2]
+
+######## dotted_import_scoped_enum_test.pyx ########
+# distutils: language = c++
+# distutils: extra_compile_args = -std=c++11
+
+
+cimport cheese
+
+cdef cheese.Cheese c = cheese.Cheese.cheddar
+assert [cheese.Cheese.cheddar, cheese.Cheese.camembert] == [1, 2]
+cdef cheese.Cheese d = int(1)
+
+######## runner.py ########
+
+import import_scoped_enum_test
+import dotted_import_scoped_enum_test
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_scoped_enums.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_scoped_enums.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_scoped_enums.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_scoped_enums.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,42 @@
+# mode: run
+# tag: cpp, cpp11
+
+cdef extern from *:
+    """
+    enum class Enum1 {
+        Item1 = 1,
+        Item2 = 2
+    };
+
+    enum class Enum2 {
+        Item4 = 4,
+        Item5 = 5
+    };
+    """
+    cpdef enum class Enum1:
+        Item1
+        Item2
+
+    cpdef enum class Enum2:
+        """Apricots and other fruits.
+        """
+        Item4
+        Item5
+
+
+def test_enum_to_list():
+    """
+    >>> test_enum_to_list()
+    """
+    assert list(Enum1) == [1, 2]
+    assert list(Enum2) == [4, 5]
+
+
+def test_enum_doc():
+    """
+    >>> Enum2.__doc__ == "Apricots and other fruits.\\n        "
+    True
+    >>> Enum1.__doc__ != "Apricots and other fruits.\\n        "
+    True
+    """
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpdef_temps_T411.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpdef_temps_T411.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpdef_temps_T411.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpdef_temps_T411.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 411
+# ticket: t411
 
 cdef class A:
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_bool.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_bool.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_bool.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_bool.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from libcpp cimport bool
 
@@ -19,3 +19,23 @@
     False
     """
     return a
+
+
+cdef bool may_raise_exception(bool value, exception) except *:
+    if exception:
+        raise exception
+    else:
+        return value
+
+def test_may_raise_exception(bool value, exception=None):
+    """
+    >>> test_may_raise_exception(False)
+    False
+    >>> test_may_raise_exception(True)
+    True
+    >>> test_may_raise_exception(True, RuntimeError)
+    Traceback (most recent call last):
+    ...
+    RuntimeError
+    """
+    return may_raise_exception(value, exception)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_bool_template_return.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_bool_template_return.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_bool_template_return.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_bool_template_return.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,16 @@
+# tag: cpp
+
+from libcpp cimport bool
+
+cdef extern from "cpp_templates_helper.h":
+    cdef cppclass BinaryAnd[T1, T2]:
+        @staticmethod
+        T1 call(T1 x, T2 y)
+
+
+def test_compound_bool_return(bool x, bool y):
+    """
+    >>> test_compound_bool_return(True, False)
+    False
+    """
+    return BinaryAnd[bool, bool].call(x, y)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_class_attrib.srctree cython-0.20.1+1~202203241016-9537/tests/run/cpp_class_attrib.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_class_attrib.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_class_attrib.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,26 @@
+# tag: cpp
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+import os
+
+example_dir = os.path.abspath(os.path.join(os.environ['CYTHON_PROJECT_DIR'],
+                              'docs/examples/userguide/wrapping_CPlusPlus'))
+
+ext_modules= cythonize(os.path.join(example_dir, "rect_with_attributes.pyx"),
+                       include_path=[example_dir])
+setup(ext_modules=ext_modules)
+
+######## runner.py ########
+
+import rect_with_attributes
+
+x0, y0, x1, y1 = 1, 2, 3, 4
+rect_obj = rect_with_attributes.PyRectangle(x0, y0, x1, y1)
+
+assert rect_obj.x0 == x0
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_classes_def.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_classes_def.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_classes_def.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_classes_def.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,11 +1,15 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, cpp11, no-cpp-locals
 # cython: experimental_cpp_class_def=True
 
 cdef double pi
 from math import pi
 from libc.math cimport sin, cos
 from libcpp cimport bool
+from libcpp.memory cimport unique_ptr
+from libcpp.vector cimport vector
+from cython.operator cimport dereference as deref
+import cython
 
 cdef extern from "shapes.h" namespace "shapes":
     cdef cppclass Shape:
@@ -20,6 +24,11 @@
     float area() const:
         cdef double theta = pi / this.n
         return this.radius * this.radius * sin(theta) * cos(theta) * this.n
+    void do_with() except *:
+        # only a compile test - the file doesn't actually have to exist
+        # "with" was broken by https://github.com/cython/cython/issues/4212
+        with open("doesnt matter") as f:
+            return
 
 def test_Poly(int n, float radius=1):
     """
@@ -143,3 +152,106 @@
     assert base.get_value() == base.value == 2 * value, base.value
 
     del base
+
+cdef cppclass Simple:
+  pass
+
+def test_default_init_no_gil():
+  with nogil:
+    s = new Simple()
+    del s
+
+
+cdef class NoisyAlloc(object):
+    cdef public name
+    def __init__(self, name):
+        print "NoisyAlloc.__init__", name
+        self.name = name
+    def __dealloc__(self):
+        try:
+            print "NoisyAlloc.__dealloc__", self.name
+        except:
+            pass  # Suppress unraisable exception warning.
+    def __repr__(self):
+        return "NoisyAlloc[%s]" % self.name
+
+cdef cppclass CppClassWithObjectMember:
+    NoisyAlloc o
+    __init__(name):
+        try:
+            print "CppClassWithObjectMember.__init__", name
+            this.o = NoisyAlloc(name)
+        except:
+            pass  # Suppress unraisable exception warning.
+    __dealloc__():
+        try:
+            print "CppClassWithObjectMember.__dealloc__", this.o.name
+        except:
+            pass  # Suppress unraisable exception warning.
+
+def test_CppClassWithObjectMember(name):
+    """
+    >>> test_CppClassWithObjectMember("gertrude")
+    CppClassWithObjectMember.__init__ gertrude
+    NoisyAlloc.__init__ gertrude
+    CppClassWithObjectMember.__dealloc__ gertrude
+    NoisyAlloc.__dealloc__ gertrude
+    """
+    x = new CppClassWithObjectMember(name)
+    del x
+
+def test_CppClassWithObjectMemberCopyAssign(name):
+    """
+    >>> test_CppClassWithObjectMemberCopyAssign("gretel")
+    CppClassWithObjectMember.__init__ gretel
+    NoisyAlloc.__init__ gretel
+    CppClassWithObjectMember.__dealloc__ gretel
+    Alive in vector NoisyAlloc[gretel]
+    CppClassWithObjectMember.__init__ leterg
+    NoisyAlloc.__init__ leterg
+    NoisyAlloc.__dealloc__ gretel
+    CppClassWithObjectMember.__dealloc__ leterg
+    Alive in vector NoisyAlloc[leterg]
+    CppClassWithObjectMember.__dealloc__ leterg
+    NoisyAlloc.__dealloc__ leterg
+    Nothing alive.
+    """
+    x = new CppClassWithObjectMember(name)
+    cdef vector[CppClassWithObjectMember] v
+    # Invokes copy constructor.
+    v.push_back(deref(x))
+    del x
+    print "Alive in vector", v[0].o
+    y = new CppClassWithObjectMember(name[::-1])
+    # Invokes copy assignment.
+    v[0] = deref(y)
+    del y
+    print "Alive in vector", v[0].o
+    v.clear()
+    print "Nothing alive."
+
+
+# Github issue #1886.
+cdef public cppclass PublicCppClassWithObjectMember:
+  object o
+
+def test_PublicCppClassWithObjectMember():
+  """
+  >>> test_PublicCppClassWithObjectMember()
+  """
+  cdef PublicCppClassWithObjectMember c
+  assert c.o is None
+
+
+cdef cppclass UncopyableConstructorArgument:
+    unique_ptr[vector[int]] member
+    __init__(unique_ptr[vector[int]] arg):
+        this.member.reset(arg.release())
+
+def test_uncopyable_constructor_argument():
+    """
+    >>> test_uncopyable_constructor_argument()
+    """
+    cdef UncopyableConstructorArgument *c = new UncopyableConstructorArgument(
+        unique_ptr[vector[int]](new vector[int]()))
+    del c
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_classes.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_classes.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_classes.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_classes.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from libcpp.vector cimport vector
 
@@ -9,7 +9,7 @@
         float area()
 
     cdef cppclass Ellipse(Shape):
-        Ellipse(int a, int b) except +
+        Ellipse(int a, int b) nogil except +
 
     cdef cppclass Circle(Ellipse):
         int radius
@@ -32,6 +32,7 @@
 
     int constructor_count, destructor_count
 
+
 def test_new_del():
     """
     >>> test_new_del()
@@ -45,6 +46,7 @@
     del rect, circ
     print constructor_count-c, destructor_count-d
 
+
 def test_default_constructor():
     """
     >>> test_default_constructor()
@@ -56,6 +58,20 @@
     finally:
         del shape
 
+
+def test_constructor_nogil():
+    """
+    >>> test_constructor_nogil()
+    True
+    """
+    with nogil:
+        shape = new Ellipse(4, 5)
+    try:
+        return 62 < shape.area() < 63 or shape.area()
+    finally:
+        del shape
+
+
 def test_rect_area(w, h):
     """
     >>> test_rect_area(3, 4)
@@ -67,6 +83,7 @@
     finally:
         del rect
 
+
 def test_overload_bint_int():
     """
     >>> test_overload_bint_int()
@@ -83,6 +100,7 @@
         del rect1
         del rect2
 
+
 def test_square_area(w):
     """
     >>> test_square_area(15)
@@ -95,6 +113,7 @@
     finally:
         del sqr
 
+
 cdef double get_area(Rectangle s):
     return s.area()
 
@@ -110,6 +129,11 @@
     finally:
         del sqr
 
+
+cdef struct StructWithEmpty:
+    Empty empty
+
+
 def get_destructor_count():
     return destructor_count
 
@@ -126,12 +150,25 @@
     print rect.method(5)
     return destructor_count
 
+def test_stack_allocation_in_struct():
+    """
+    >>> d = test_stack_allocation_in_struct()
+    >>> get_destructor_count() - d
+    1
+    """
+    cdef StructWithEmpty swe
+    sizeof(swe.empty) # use it for something
+    return destructor_count
+
 cdef class EmptyHolder:
     cdef Empty empty
 
 cdef class AnotherEmptyHolder(EmptyHolder):
     cdef Empty another_empty
 
+cdef class EmptyViaStructHolder:
+    cdef StructWithEmpty swe
+
 def test_class_member():
     """
     >>> test_class_member()
@@ -148,6 +185,7 @@
     assert destructor_count - start_destructor_count == 2, \
            destructor_count - start_destructor_count
 
+
 def test_derived_class_member():
     """
     >>> test_derived_class_member()
@@ -161,6 +199,19 @@
     assert destructor_count - start_destructor_count == 2, \
            destructor_count - start_destructor_count
 
+def test_class_in_struct_member():
+    """
+    >>> test_class_in_struct_member()
+    """
+    start_constructor_count = constructor_count
+    start_destructor_count = destructor_count
+    e = EmptyViaStructHolder()
+    #assert constructor_count - start_constructor_count == 1, \
+    #       constructor_count - start_constructor_count
+    del e
+    assert destructor_count - start_destructor_count == 1, \
+           destructor_count - start_destructor_count
+
 cdef class TemplateClassMember:
     cdef vector[int] x
     cdef vector[vector[Empty]] vec
@@ -179,3 +230,34 @@
     del o
     assert destructor_count - start_destructor_count == 2, \
            destructor_count - start_destructor_count
+
+
+ctypedef vector[int]* vector_int_ptr
+cdef vector[vector_int_ptr] create_to_delete() except *:
+    cdef vector[vector_int_ptr] v
+    v.push_back(new vector[int]())
+    return v
+cdef int f(int x):
+    return x
+
+
+def test_nested_del():
+    """
+    >>> test_nested_del()
+    """
+    cdef vector[vector_int_ptr] v
+    v.push_back(new vector[int]())
+    del v[0]
+    del create_to_delete()[f(f(0))]
+
+
+def test_nested_del_repeat():
+    """
+    >>> test_nested_del_repeat()
+    """
+    cdef vector[vector_int_ptr] v
+    v.push_back(new vector[int]())
+    del v[0]
+    del create_to_delete()[f(f(0))]
+    del create_to_delete()[f(f(0))]
+    del create_to_delete()[f(f(0))]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_class_redef.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_class_redef.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_class_redef.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_class_redef.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,7 @@
-# tag: cpp
+# mode: run
+# tag: cpp, warnings, no-cpp-locals
 
-# This gives a warning, but should not give an error.
+# This gives a warning about the previous .pxd definition, but should not give an error.
 cdef cppclass Foo:
     int _foo
     int get_foo():
@@ -20,3 +21,8 @@
         return foo.get_foo()
     finally:
         del foo
+
+
+_WARNINGS = """
+5:5: 'Foo' already defined  (ignoring second definition)
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_const_method.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_const_method.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_const_method.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_const_method.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 # cython: experimental_cpp_class_def=True
 
 from libcpp.vector cimport vector
@@ -82,6 +82,6 @@
 cdef vector_members(vector[const Wrapper[int]*] a, const vector[wrapInt*] b):
     # TODO: Cython-level error.
     # b[0].set(100)
-    
+
     # TODO: const_iterator
     return [x.get() for x in a], b[0].get()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_enums.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_enums.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_enums.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_enums.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,58 @@
+# tag: cpp
+# mode: run, no-cpp-locals
+
+cdef extern from *:
+    """
+    enum Enum1 {
+        Item1,
+        Item2
+    };
+
+    """
+    cdef enum Enum1:
+        Item1
+        Item2
+
+a = Item1
+b = Item2
+
+cdef Enum1 x, y
+x = Item1
+y = Item2
+
+
+def compare_enums():
+    """
+    >>> compare_enums()
+    (True, True, True, True)
+    """
+    return x == a, a == Item1, b == y, y == Item2
+
+
+cdef extern from * namespace "Namespace1":
+    """
+    namespace Namespace1 {
+        enum Enum2 {
+            Item3,
+            Item4
+        };
+    }
+    """
+    cdef enum Enum2:
+        Item3
+        Item4
+
+c = Item3
+d = Item4
+
+cdef Enum2 z, w
+z = Item3
+w = Item4
+
+
+def compare_namespace_enums():
+    """
+    >>> compare_namespace_enums()
+    (True, True, True, True)
+    """
+    return z == c, c == Item3, d == w, d == Item4
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_exception_declaration_compatibility.srctree cython-0.20.1+1~202203241016-9537/tests/run/cpp_exception_declaration_compatibility.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_exception_declaration_compatibility.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_exception_declaration_compatibility.srctree	1970-01-01 00:00:00.000000000 +0000
@@ -1,38 +0,0 @@
-# tag: cpp
-
-"""
-PYTHON setup.py build_ext -i
-PYTHON test.py
-"""
-
-############### setup.py ###################
-from distutils.core import setup
-from Cython.Build import cythonize
-
-setup(
-    name="cython_test",
-    ext_modules=cythonize('*.pyx', language="c++")
-)
-
-
-############### test.py ###################
-
-from cpp_exc import TestClass
-
-TestClass().test_func()
-
-
-############### cpp_exc.pxd ###################
-
-cdef inline void handle_exception():
-    pass
-
-cdef class TestClass:
-    cpdef test_func(self) except +handle_exception
-
-
-############### cpp_exc.pyx ###################
-
-cdef class TestClass:
-    cpdef test_func(self) except +handle_exception:
-        print('test')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions_helper.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions_helper.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,4 @@
+#include 
 #include 
 #include 
 #include 
@@ -61,3 +62,19 @@
 void raise_underflow() {
     throw std::underflow_error("underflow_error");
 }
+
+PyObject *raise_or_throw(int py) {
+    if (!py) {
+        throw std::runtime_error("oopsie");
+    }
+    PyErr_SetString(PyExc_ValueError, "oopsie");
+    return NULL;
+}
+
+int raise_or_throw_int(int py) {
+    if (!py) {
+        throw std::runtime_error("oopsie");
+    }
+    PyErr_SetString(PyExc_ValueError, "oopsie");
+    return -1;
+}
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -21,6 +21,9 @@
     cdef void raise_typeerror() except +
     cdef void raise_underflow() except +
 
+    cdef raise_or_throw(bint py) except +
+    cdef int raise_or_throw_int(bint py) except +*
+
     cdef cppclass Foo:
         int bar_raw "bar"(bint fire) except +
         int bar_value "bar"(bint fire) except +ValueError
@@ -98,6 +101,32 @@
     """
     raise_underflow()
 
+def test_func_that_can_raise_or_throw(bint py):
+    """
+    >>> test_func_that_can_raise_or_throw(0)
+    Traceback (most recent call last):
+    ...
+    RuntimeError: oopsie
+    >>> test_func_that_can_raise_or_throw(1)
+    Traceback (most recent call last):
+    ...
+    ValueError: oopsie
+    """
+    raise_or_throw(py)
+
+def test_func_that_can_raise_or_throw_c_return(bint py):
+    """
+    >>> test_func_that_can_raise_or_throw_c_return(0)
+    Traceback (most recent call last):
+    ...
+    RuntimeError: oopsie
+    >>> test_func_that_can_raise_or_throw_c_return(1)
+    Traceback (most recent call last):
+    ...
+    ValueError: oopsie
+    """
+    raise_or_throw_int(py)
+
 def test_int_raw(bint fire):
     """
     >>> test_int_raw(False)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions_utility_code.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions_utility_code.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_exceptions_utility_code.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_exceptions_utility_code.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,33 @@
+# mode: run
+# tag: cpp, werror, no-cpp-locals
+# ticket: 3065
+
+# This is intentionally in a file on its own. The issue was that it failed to generate utility-code
+# and so putting it with the other c++ exception checks wouldn't be a useful test
+
+cdef extern from *:
+    """
+    #include 
+
+    void cppf(int raiseCpp) {
+        if (raiseCpp) {
+            throw std::runtime_error("cpp");
+        } else {
+            PyErr_SetString(PyExc_RuntimeError, "py");
+        }
+    }
+    """
+    void cppf(int) except+*
+
+
+def callcppf(int raiseCpp):
+    """
+    >>> callcppf(0)
+    py
+    >>> callcppf(1)
+    cpp
+    """
+    try:
+        cppf(raiseCpp)
+    except RuntimeError as e:
+        print(e.args[0])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_forwarding_ref.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_forwarding_ref.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_forwarding_ref.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_forwarding_ref.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,66 @@
+# mode: run
+# tag: cpp, cpp11, no-cpp-locals
+
+from libcpp.utility cimport move
+
+
+cdef extern from *:
+    """
+    #include 
+
+    const char* f(int& x) {
+        (void) x;
+        return "lvalue-ref";
+    }
+
+    const char* f(int&& x) {
+        (void) x;
+        return "rvalue-ref";
+    }
+
+    template 
+    const char* foo(T&& x)
+    {
+        return f(std::forward(x));
+    }
+    """
+    const char* foo[T](T&& x)
+
+
+cdef extern from *:
+    """
+    #include 
+
+    template 
+    const char* bar(T1 x, T1 y) {
+        return "first";
+    }
+
+    template 
+    const char* bar(T1&& x, T2 y, T2 z) {
+        return "second";
+    }
+
+    """
+    const char* bar[T1](T1 x, T1 y)
+    const char* bar[T1, T2](T1&& x, T2 y, T2 z)
+
+def test_forwarding_ref():
+    """
+    >>> test_forwarding_ref()
+    """
+    cdef int x = 1
+    assert foo(x) == b"lvalue-ref"
+    assert foo((1)) == b"rvalue-ref"
+    assert foo(move(x)) == b"rvalue-ref"
+
+
+def test_forwarding_ref_overload():
+    """
+    >>> test_forwarding_ref_overload()
+    """
+    cdef int x = 1
+    cdef int y = 2
+    cdef int z = 3
+    assert bar(x, y) == b"first"
+    assert bar(x, y, z) == b"second"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.cpp cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.cpp
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.cpp	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.cpp	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,50 @@
+#include "cpp_function_lib.h"
+
+double add_one(double a, int b)
+{
+    return a + (double) b + 1.0;
+}
+
+double add_two(double a, int b)
+{
+    return a + (double) b + 2.0;
+}
+
+
+AddAnotherFunctor::AddAnotherFunctor(double to_add)
+    : to_add(to_add)
+{
+}
+
+double AddAnotherFunctor::operator()(double a, int b) const
+{
+    return a + (double) b + this->to_add;
+};
+
+
+FunctionKeeper::FunctionKeeper(std::function user_function)
+    : my_function(user_function)
+{
+}
+
+FunctionKeeper::~FunctionKeeper()
+{
+}
+
+void FunctionKeeper::set_function(std::function user_function)
+{
+    this->my_function = user_function;
+}
+
+std::function FunctionKeeper::get_function() const
+{
+    return this->my_function;
+}
+
+double FunctionKeeper::call_function(double a, int b) const
+{
+    if (!this->my_function) {
+        throw std::runtime_error("Trying to call undefined function!");
+    }
+    return this->my_function(a, b);
+};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.h	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.h	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,35 @@
+#ifndef CPP_FUNCTION_LIB_H
+#define CPP_FUNCTION_LIB_H
+
+#include 
+
+// Functions, functor and a holder of std::function used by cpp_stl_function.pyx tests.
+
+double add_one(double a, int b);
+double add_two(double a, int b);
+
+class AddAnotherFunctor
+{
+    double to_add;
+
+public:
+    AddAnotherFunctor(double to_add);
+    double operator()(double a, int b) const;
+};
+
+
+class FunctionKeeper
+{
+    std::function my_function;
+
+public:
+    FunctionKeeper(std::function user_function);
+    virtual ~FunctionKeeper();
+
+    void set_function(std::function user_function);
+    std::function get_function() const;
+
+    double call_function(double a, int b) const;
+};
+
+#endif
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.pxd cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_function_lib.pxd	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_function_lib.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,19 @@
+from libcpp.functional cimport function
+
+cdef extern from "cpp_function_lib.cpp":
+    # CPP is include here so that it doesn't need to be compiled externally
+    pass
+
+cdef extern from "cpp_function_lib.h":
+    double add_one(double, int)
+    double add_two(double a, int b)
+
+    cdef cppclass AddAnotherFunctor:
+        AddAnotherFunctor(double to_add)
+        double call "operator()"(double a, int b)
+
+    cdef cppclass FunctionKeeper:
+        FunctionKeeper(function[double(double, int)] user_function)
+        void set_function(function[double(double, int)] user_function)
+        function[double(double, int)] get_function()
+        double call_function(double a, int b) except +
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_iterators.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_iterators.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_iterators.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_iterators.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,7 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
+from libcpp.deque cimport deque
 from libcpp.vector cimport vector
 from cython.operator cimport dereference as deref
 
@@ -15,13 +16,59 @@
     >>> test_vector([1, 2, 3])
     [1, 2, 3]
     """
-    cdef vector[int] v = py_v
+    cdef vector[int] vint = py_v
     cdef vector[int] result
     with nogil:
-        for item in v:
+        for item in vint:
             result.push_back(item)
     return result
 
+def test_deque_iterator_subtraction(py_v):
+    """
+    >>> print(test_deque_iterator_subtraction([1, 2, 3]))
+    3
+    """
+    cdef deque[int] dint
+    for i in py_v:
+        dint.push_back(i)
+    cdef deque[int].iterator first = dint.begin()
+    cdef deque[int].iterator last = dint.end()
+
+    return last - first
+
+def test_vector_iterator_subtraction(py_v):
+    """
+    >>> print(test_vector_iterator_subtraction([1, 2, 3]))
+    3
+    """
+    cdef vector[int] vint = py_v
+    cdef vector[int].iterator first = vint.begin()
+    cdef vector[int].iterator last = vint.end()
+
+    return last - first
+
+def test_deque_iterator_addition(py_v):
+    """
+    >>> test_deque_iterator_addition([2, 4, 6])
+    6
+    """
+    cdef deque[int] dint
+    for i in py_v:
+        dint.push_back(i)
+    cdef deque[int].iterator first = dint.begin()
+
+    return deref(first+2)
+
+def test_vector_iterator_addition(py_v):
+    """
+    >>> test_vector_iterator_addition([2, 4, 6])
+    6
+    """
+    cdef vector[int] vint = py_v
+    cdef vector[int].iterator first = vint.begin()
+
+    return deref(first+2)
+
 def test_ptrs():
     """
     >>> test_ptrs()
@@ -93,3 +140,40 @@
         if vint is not orig_vint:
             del vint
         del orig_vint
+
+cdef extern from *:
+    """
+    std::vector make_vec1() {
+        std::vector vint;
+        vint.push_back(1);
+        vint.push_back(2);
+        return vint;
+    }
+    """
+    cdef vector[int] make_vec1() except +
+
+cdef vector[int] make_vec2() except *:
+    return make_vec1()
+
+cdef vector[int] make_vec3():
+    try:
+        return make_vec1()
+    except:
+        pass
+
+def test_iteration_from_function_call():
+    """
+    >>> test_iteration_from_function_call()
+    1
+    2
+    1
+    2
+    1
+    2
+    """
+    for i in make_vec1():
+        print(i)
+    for i in make_vec2():
+        print(i)
+    for i in make_vec3():
+        print(i)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_directive.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_directive.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_directive.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_directive.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,235 @@
+# mode: run
+# tag: cpp, cpp17, no-cpp-locals
+# no-cpp-locals because the test is already run with it explicitly set
+
+# cython: cpp_locals=True
+
+cimport cython
+
+from libcpp cimport bool as cppbool
+
+cdef extern from *:
+    r"""
+    static void print_C_destructor();
+
+    class C {
+        public:
+            C() = delete; // look! No default constructor
+            C(int x, bool print_destructor=true) : x(x), print_destructor(print_destructor) {}
+            C(C&& rhs) : x(rhs.x), print_destructor(rhs.print_destructor) {
+                rhs.print_destructor = false; // moved-from instances are deleted silently
+            }
+            C& operator=(C&& rhs) {
+                x=rhs.x;
+                print_destructor=rhs.print_destructor;
+                rhs.print_destructor = false; // moved-from instances are deleted silently
+                return *this;
+            }
+            C(const C& rhs) = default;
+            C& operator=(const C& rhs) = default;
+            ~C() {
+                if (print_destructor) print_C_destructor();
+            }
+
+            int getX() const { return x; }
+
+        private:
+            int x;
+            bool print_destructor;
+    };
+
+    C make_C(int x) {
+        return C(x);
+    }
+    """
+    cdef cppclass C:
+        C(int)
+        C(int, cppbool)
+        int getX() const
+    C make_C(int) except +  # needs a temp to receive
+
+# this function just makes sure the output from the destructor can be captured by doctest
+cdef void print_C_destructor "print_C_destructor" () with gil:
+    print("~C()")
+
+def maybe_assign_infer(assign, value, do_print):
+    """
+    >>> maybe_assign_infer(True, 5, True)
+    5
+    ~C()
+    >>> maybe_assign_infer(False, 0, True)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    >>> maybe_assign_infer(False, 0, False)  # no destructor call here
+    """
+    if assign:
+        x = C(value)
+    if do_print:
+        print(x.getX())
+
+def maybe_assign_cdef(assign, value):
+    """
+    >>> maybe_assign_cdef(True, 5)
+    5
+    ~C()
+    >>> maybe_assign_cdef(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    cdef C x
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_annotation(assign, value):
+    """
+    >>> maybe_assign_annotation(True, 5)
+    5
+    ~C()
+    >>> maybe_assign_annotation(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    x: C
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_directive1(assign, value):
+    """
+    >>> maybe_assign_directive1(True, 5)
+    5
+    ~C()
+    >>> maybe_assign_directive1(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    x = cython.declare(C)
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+@cython.locals(x=C)
+def maybe_assign_directive2(assign, value):
+    """
+    >>> maybe_assign_directive2(True, 5)
+    5
+    ~C()
+    >>> maybe_assign_directive2(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_nocheck(assign, value):
+    """
+    >>> maybe_assign_nocheck(True, 5)
+    5
+    ~C()
+
+    # unfortunately it's quite difficult to test not assigning because there's a decent chance it'll crash
+    """
+    if assign:
+        x = C(value)
+    with cython.initializedcheck(False):
+        print(x.getX())
+
+def uses_temp(value):
+    """
+    needs a temp to handle the result of make_C - still doesn't use the default constructor
+    >>> uses_temp(10)
+    10
+    ~C()
+    """
+
+    x = make_C(value)
+    print(x.getX())
+
+# c should not be optional - it isn't easy to check this, but we can at least check it compiles
+cdef void has_argument(C c):
+    print(c.getX())
+
+def call_has_argument():
+    """
+    >>> call_has_argument()
+    50
+    """
+    has_argument(C(50, False))
+
+cdef class HoldsC:
+    """
+    >>> inst = HoldsC(True, False)
+    >>> inst.getCX()
+    10
+    >>> access_from_function_with_different_directive(inst)
+    10
+    10
+    >>> inst.getCX()  # it was changed in access_from_function_with_different_directive
+    20
+    >>> inst = HoldsC(False, False)
+    >>> inst.getCX()
+    Traceback (most recent call last):
+        ...
+    AttributeError: C++ attribute 'value' is not initialized
+    >>> access_from_function_with_different_directive(inst)
+    Traceback (most recent call last):
+        ...
+    AttributeError: C++ attribute 'value' is not initialized
+    """
+    cdef C value
+    def __cinit__(self, initialize, print_destructor):
+        if initialize:
+            self.value = C(10, print_destructor)
+
+    def getCX(self):
+        return self.value.getX()
+
+cdef acceptC(C& c):
+    return c.getX()
+
+@cython.cpp_locals(False)
+def access_from_function_with_different_directive(HoldsC c):
+    # doctest is in HoldsC class
+    print(acceptC(c.value))  # this originally tried to pass a __Pyx_Optional as a C instance
+    print(c.value.getX())
+    c.value = C(20, False) # make sure that we can change it too
+
+def dont_test_on_pypy(f):
+    import sys
+    if not hasattr(sys, "pypy_version_info"):
+        return f
+
+@dont_test_on_pypy  # non-deterministic destruction
+def testHoldsCDestruction(initialize):
+    """
+    >>> testHoldsCDestruction(True)
+    ~C()
+    >>> testHoldsCDestruction(False)  # no destructor
+    """
+    x = HoldsC(initialize, True)
+    del x
+
+cdef C global_var
+
+def initialize_global_var():
+    global global_var
+    global_var = C(-1, False)
+
+def read_global_var():
+    """
+    >>> read_global_var()
+    Traceback (most recent call last):
+        ...
+    NameError: C++ global 'global_var' is not initialized
+    >>> initialize_global_var()
+    >>> read_global_var()
+    -1
+    """
+    print(global_var.getX())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_directive_unused.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_directive_unused.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_directive_unused.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_directive_unused.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,14 @@
+# mode: run
+# tag: cpp, cpp17, no-cpp-locals
+# no cpp_locals because this test is already run with cpp_locals explicitly set
+
+# cython: cpp_locals=True
+
+cdef cppclass C:
+    C()
+
+cdef class PyC:
+    """
+    >>> PyC() and None  # doesn't really do anything, but should run
+    """
+    cdef C  # this limited usage wasn't triggering the creation of utility code
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_parallel.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_parallel.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_locals_parallel.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_locals_parallel.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,33 @@
+# mode: run
+# tag: cpp, cpp17, no-cpp-locals, openmp
+# no-cpp-locals because the test is already run with it explicitly set
+
+# cython: cpp_locals=True
+
+from cython.parallel cimport prange
+
+cdef extern from *:
+    """
+    class Test {
+    public:
+        Test() = delete;
+        Test(int v) : value(v) {}
+
+        int get_value() const { return value; }
+    private:
+        int value;
+    };
+    """
+    cdef cppclass Test:
+        Test(int) nogil
+        int get_value()
+
+def test():
+    """
+    >>> test()
+    9
+    """
+    cdef int i
+    for i in prange(10, nogil=True):
+        var = Test(i)
+    print(var.get_value())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_move.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_move.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_move.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_move.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,34 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+from libcpp cimport nullptr
+from libcpp.memory cimport shared_ptr, make_shared
+from libcpp.utility cimport move
+from cython.operator cimport dereference
+
+cdef extern from *:
+    """
+    #include 
+
+    template const char* move_helper(T&) { return "lvalue-ref"; }
+    template const char* move_helper(T&&) { return "rvalue-ref"; }
+    """
+    const char* move_helper[T](T)
+
+def test_move_assignment():
+    """
+    >>> test_move_assignment()
+    """
+    cdef shared_ptr[int] p1, p2
+    p1 = make_shared[int](1337)
+    p2 = move(p1)
+    assert p1 == nullptr
+    assert dereference(p2) == 1337
+
+def test_move_func_call():
+    """
+    >>> test_move_func_call()
+    """
+    cdef shared_ptr[int] p
+    assert move_helper(p) == b'lvalue-ref'
+    assert move_helper(move(p)) == b'rvalue-ref'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_nested_classes.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_nested_classes.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_nested_classes.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_nested_classes.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 cdef extern from "cpp_nested_classes_support.h":
     cdef cppclass A:
@@ -13,12 +13,23 @@
 
     cdef cppclass TypedClass[T]:
         ctypedef T MyType
+        struct MyStruct:
+            T typed_value
+            int int_value
         union MyUnion:
             T typed_value
             int int_value
         enum MyEnum:
             value
 
+    cdef cppclass SpecializedTypedClass(TypedClass[double]):
+        pass
+
+
+ctypedef A AliasA1
+ctypedef AliasA1 AliasA2
+
+
 def test_nested_classes():
     """
     >>> test_nested_classes()
@@ -40,6 +51,20 @@
     cdef A.my_int x = py_x
     assert A.negate(x) == -py_x
 
+def test_typedef_for_nested(py_x):
+    """
+    >>> test_typedef_for_nested(5)
+    """
+    cdef AliasA1.my_int x = py_x
+    assert A.negate(x) == -py_x
+
+def test_typedef_for_nested_deep(py_x):
+    """
+    >>> test_typedef_for_nested_deep(5)
+    """
+    cdef AliasA2.my_int x = py_x
+    assert A.negate(x) == -py_x
+
 def test_typed_nested_typedef(x):
     """
     >>> test_typed_nested_typedef(4)
@@ -66,3 +91,53 @@
     assert u.int_value == x
     u.typed_value = x
     return u.typed_value
+
+def test_nested_struct(x):
+    """
+    >>> test_nested_struct(2)
+    2.0
+    """
+    cdef TypedClass[double].MyStruct s
+    s.int_value = x
+    assert s.int_value == x
+    s.typed_value = x
+    return s.typed_value
+
+
+
+def test_typed_nested_sub_typedef(x):
+    """
+    >>> test_typed_nested_sub_typedef(4)
+    4.0
+    """
+    cdef SpecializedTypedClass.MyType dx = x
+    return dx
+
+def test_nested_sub_enum(SpecializedTypedClass.MyEnum x):
+    """
+    >>> test_nested_sub_enum(4)
+    False
+    """
+    return x == 0
+
+def test_nested_sub_union(x):
+    """
+    >>> test_nested_sub_union(2)
+    2.0
+    """
+    cdef SpecializedTypedClass.MyUnion u
+    u.int_value = x
+    assert u.int_value == x
+    u.typed_value = x
+    return u.typed_value
+
+def test_nested_sub_struct(x):
+    """
+    >>> test_nested_sub_struct(2)
+    2.0
+    """
+    cdef SpecializedTypedClass.MyStruct s
+    s.int_value = x
+    assert s.int_value == x
+    s.typed_value = x
+    return s.typed_value
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_nested_classes_support.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_nested_classes_support.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_nested_classes_support.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_nested_classes_support.h	2022-03-24 10:16:46.000000000 +0000
@@ -27,5 +27,11 @@
     T typed_value;
     int int_value;
   };
+  struct MyStruct {
+    T typed_value;
+    int int_value;
+  };
   typedef T MyType;
 };
+
+class SpecializedTypedClass : public TypedClass {};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_nonstdint.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_nonstdint.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_nonstdint.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_nonstdint.h	2022-03-24 10:16:46.000000000 +0000
@@ -53,7 +53,7 @@
         resize_signed_int(bytes, N, extended, len);
     }
     bool res = memcmp(extended, other, len);
-    delete extended;
+    delete [] extended;
     return res;
   }
   bool operator!=(const long long val) const
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_nonstdint.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_nonstdint.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_nonstdint.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_nonstdint.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 cdef extern from "cpp_nonstdint.h":
     ctypedef int Int24
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_operator_exc_handling_helper.hpp cython-0.20.1+1~202203241016-9537/tests/run/cpp_operator_exc_handling_helper.hpp
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_operator_exc_handling_helper.hpp	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_operator_exc_handling_helper.hpp	2022-03-24 10:16:46.000000000 +0000
@@ -201,3 +201,16 @@
     return *this;
   }
 };
+
+class second_call_is_different {
+    int count;
+    public:
+        second_call_is_different(): count(0) {}
+    bool operator<(const second_call_is_different& lhs) {
+        if (count>0) {
+            return true;
+        }
+        ++count;
+        return false;
+    }
+};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_operator_exc_handling.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_operator_exc_handling.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_operator_exc_handling.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_operator_exc_handling.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator import (preincrement, predecrement,
                              postincrement, postdecrement)
@@ -42,6 +42,10 @@
         wrapped_int &operator=(const wrapped_int &other) except +ArithmeticError
         wrapped_int &operator=(const long long &vao) except +
 
+    cdef cppclass second_call_is_different:
+        second_call_is_different()
+        bool operator<(const second_call_is_different&) except +
+
 
 def assert_raised(f, *args, **kwargs):
     err = kwargs.get('err', None)
@@ -268,3 +272,13 @@
     assert_raised(separate_exceptions, 4, 1, 1, 1, 3, err=ArithmeticError)
     assert_raised(call_temp_separation, 2, 1, 4, err=AttributeError)
     assert_raised(call_temp_separation, 2, 4, 1, err=IndexError)
+
+def test_only_single_call():
+    """
+    Previous version of the operator handling code called the operator twice
+    (Resulting in a crash)
+    >>> test_only_single_call()
+    False
+    """
+    cdef second_call_is_different inst
+    return inst>)
+
+NONMEMBER_BIN_OP(|)
+NONMEMBER_BIN_OP(&)
+NONMEMBER_BIN_OP(^)
+NONMEMBER_BIN_OP(COMMA)
+
+NONMEMBER_BIN_OP2(+)
+NONMEMBER_BIN_OP2(-)
+NONMEMBER_BIN_OP2(*)
+NONMEMBER_BIN_OP2(/)
+NONMEMBER_BIN_OP2(%)
+
+NONMEMBER_BIN_OP2(<<)
+NONMEMBER_BIN_OP2(>>)
+
+NONMEMBER_BIN_OP2(|)
+NONMEMBER_BIN_OP2(&)
+NONMEMBER_BIN_OP2(^)
+NONMEMBER_BIN_OP2(COMMA)
+
+
+/* RefTestOps */
+
+#define REF_UN_OP(op) int& operator op () { return value; }
+#define REF_POST_UN_OP(op) int& operator op (int x) { x++; return value; }
+#define REF_BIN_OP(op) int& operator op (int x) { x++; return value; }
+
+class RefTestOps {
+    int value;
+
+public:
+
+    RefTestOps() { value = 0; }
+
+    REF_UN_OP(-);
+    REF_UN_OP(+);
+    REF_UN_OP(*);
+    REF_UN_OP(~);
+    REF_UN_OP(!);
+    REF_UN_OP(&);
+
+    REF_UN_OP(++);
+    REF_UN_OP(--);
+    REF_POST_UN_OP(++);
+    REF_POST_UN_OP(--);
+
+    REF_BIN_OP(+);
+    REF_BIN_OP(-);
+    REF_BIN_OP(*);
+    REF_BIN_OP(/);
+    REF_BIN_OP(%);
+
+    REF_BIN_OP(<<);
+    REF_BIN_OP(>>);
+
+    REF_BIN_OP(|);
+    REF_BIN_OP(&);
+    REF_BIN_OP(^);
+    REF_BIN_OP(COMMA);
+
+    REF_BIN_OP(==);
+    REF_BIN_OP(!=);
+    REF_BIN_OP(<=);
+    REF_BIN_OP(<);
+    REF_BIN_OP(>=);
+    REF_BIN_OP(>);
+
+    REF_BIN_OP([]);
+    REF_BIN_OP(());
+};
+
+
+/* TruthClass */
+
 class TruthClass {
 public:
   TruthClass() : value(false) {}
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_operators.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_operators.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_operators.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_operators.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,8 @@
 # mode: run
 # tag: cpp, werror
 
+from __future__ import division
+
 from cython cimport typeof
 
 cimport cython.operator
@@ -9,37 +11,56 @@
 from libc.string cimport const_char
 from libcpp cimport bool
 
+
 cdef out(s, result_type=None):
     print '%s [%s]' % (s.decode('ascii'), result_type)
 
-cdef extern from "cpp_operators_helper.h":
+
+cdef iout(int s, result_type=None):
+    print '%s [%s]' % (s, result_type)
+
+
+cdef extern from "cpp_operators_helper.h" nogil:
     cdef cppclass TestOps:
 
-        const_char* operator+()
-        const_char* operator-()
-        const_char* operator*()
-        const_char* operator~()
-        const_char* operator!()
+        const_char* operator+() except +
+        const_char* operator-() except +
+        const_char* operator*() except +
+        const_char* operator~() except +
+        const_char* operator!() except +
 
+        # FIXME: using 'except +' here leads to wrong calls ???
         const_char* operator++()
         const_char* operator--()
         const_char* operator++(int)
         const_char* operator--(int)
 
-        const_char* operator+(int)
-        const_char* operator-(int)
-        const_char* operator*(int)
-        const_char* operator/(int)
-        const_char* operator%(int)
-
-        const_char* operator|(int)
-        const_char* operator&(int)
-        const_char* operator^(int)
-        const_char* operator,(int)
-
-        const_char* operator<<(int)
-        const_char* operator>>(int)
+        const_char* operator+(int) except +
+        const_char* operator+(int,const TestOps&) except +
+        const_char* operator-(int) except +
+        const_char* operator-(int,const TestOps&) except +
+        const_char* operator*(int) except +
+        # deliberately omitted operator* to test case where only defined outside class
+        const_char* operator/(int) except +
+        const_char* operator/(int,const TestOps&) except +
+        const_char* operator%(int) except +
+        const_char* operator%(int,const TestOps&) except +
+
+        const_char* operator|(int) except +
+        const_char* operator|(int,const TestOps&) except +
+        const_char* operator&(int) except +
+        const_char* operator&(int,const TestOps&) except +
+        const_char* operator^(int) except +
+        const_char* operator^(int,const TestOps&) except +
+        const_char* operator,(int) except +
+        const_char* operator,(int,const TestOps&) except +
+
+        const_char* operator<<(int) except +
+        const_char* operator<<(int,const TestOps&) except +
+        const_char* operator>>(int) except +
+        const_char* operator>>(int,const TestOps&) except +
 
+        # FIXME: using 'except +' here leads to invalid C++ code ???
         const_char* operator==(int)
         const_char* operator!=(int)
         const_char* operator>=(int)
@@ -47,8 +68,73 @@
         const_char* operator>(int)
         const_char* operator<(int)
 
-        const_char* operator[](int)
-        const_char* operator()(int)
+        const_char* operator[](int) except +
+        const_char* operator()(int) except +
+
+    # Defining the operator outside the class does work
+    # but doesn't help when importing from pxd files
+    # (they don't get imported)
+    const_char* operator+(float,const TestOps&) except +
+    # deliberately omitted operator- to test case where only defined in class
+    const_char* operator*(float,const TestOps&) except +
+    const_char* operator/(float,const TestOps&) except +
+    const_char* operator%(float,const TestOps&) except +
+
+    const_char* operator|(float,const TestOps&) except +
+    const_char* operator&(float,const TestOps&) except +
+    const_char* operator^(float,const TestOps&) except +
+    const_char* operator,(float,const TestOps&) except +
+
+    const_char* operator<<(float,const TestOps&) except +
+    const_char* operator>>(float,const TestOps&) except +
+
+    cdef cppclass RefTestOps:
+
+        int& operator+() except +
+        int& operator-() except +
+        int& operator*() except +
+        int& operator~() except +
+        int& operator!() except +
+
+        int& operator++() except +
+        int& operator--() except +
+        int& operator++(int) except +
+        int& operator--(int) except +
+
+        int& operator+(int) except +
+        int& operator+(int,const TestOps&) except +
+        int& operator-(int) except +
+        int& operator-(int,const TestOps&) except +
+        int& operator*(int) except +
+        # deliberately omitted operator* to test case where only defined outside class
+        int& operator/(int) except +
+        int& operator/(int,const TestOps&) except +
+        int& operator%(int) except +
+        int& operator%(int,const TestOps&) except +
+
+        int& operator|(int) except +
+        int& operator|(int,const TestOps&) except +
+        int& operator&(int) except +
+        int& operator&(int,const TestOps&) except +
+        int& operator^(int) except +
+        int& operator^(int,const TestOps&) except +
+        int& operator,(int) except +
+        int& operator,(int,const TestOps&) except +
+
+        int& operator<<(int) except +
+        int& operator<<(int,const TestOps&) except +
+        int& operator>>(int) except +
+        int& operator>>(int,const TestOps&) except +
+
+        int& operator==(int) except +
+        int& operator!=(int) except +
+        int& operator>=(int) except +
+        int& operator<=(int) except +
+        int& operator>(int) except +
+        int& operator<(int) except +
+
+        int& operator[](int) except +
+        int& operator()(int) except +
 
     cdef cppclass TruthClass:
         TruthClass()
@@ -56,9 +142,11 @@
         bool operator bool()
         bool value
 
+
 cdef cppclass TruthSubClass(TruthClass):
     pass
 
+
 def test_unops():
     """
     >>> test_unops()
@@ -129,6 +217,63 @@
     out(x, typeof(x))
     del t
 
+def test_nonmember_binop():
+    """
+    >>> test_nonmember_binop()
+    nonmember binary + [const_char *]
+    nonmember binary - [const_char *]
+    nonmember binary / [const_char *]
+    nonmember binary % [const_char *]
+    nonmember binary & [const_char *]
+    nonmember binary | [const_char *]
+    nonmember binary ^ [const_char *]
+    nonmember binary << [const_char *]
+    nonmember binary >> [const_char *]
+    nonmember binary COMMA [const_char *]
+    nonmember binary2 + [const_char *]
+    nonmember binary2 * [const_char *]
+    nonmember binary2 / [const_char *]
+    nonmember binary2 % [const_char *]
+    nonmember binary2 & [const_char *]
+    nonmember binary2 | [const_char *]
+    nonmember binary2 ^ [const_char *]
+    nonmember binary2 << [const_char *]
+    nonmember binary2 >> [const_char *]
+    nonmember binary2 COMMA [const_char *]
+    """
+
+    cdef TestOps* t = new TestOps()
+    out(1 + t[0], typeof(1 + t[0]))
+    out(1 - t[0], typeof(1 - t[0]))
+    # * deliberately omitted
+    out(1 / t[0], typeof(1 / t[0]))
+    out(1 % t[0], typeof(1 % t[0]))
+    out(1 & t[0], typeof(1 & t[0]))
+    out(1 | t[0], typeof(1 | t[0]))
+    out(1 ^ t[0], typeof(1 ^ t[0]))
+    out(1 << t[0], typeof(1 << t[0]))
+    out(1 >> t[0], typeof(1 >> t[0]))
+
+    x = cython.operator.comma(1, t[0])
+    out(x, typeof(x))
+
+    # now test float operators defined outside class
+    out(1. + t[0], typeof(1. + t[0]))
+    # operator - deliberately omitted
+    out(1. * t[0], typeof(1. * t[0]))
+    out(1. / t[0], typeof(1. / t[0]))
+    out(1. % t[0], typeof(1. % t[0]))
+    out(1. & t[0], typeof(1. & t[0]))
+    out(1. | t[0], typeof(1. | t[0]))
+    out(1. ^ t[0], typeof(1. ^ t[0]))
+    out(1. << t[0], typeof(1. << t[0]))
+    out(1. >> t[0], typeof(1. >> t[0]))
+
+    # for some reason we need a cdef here - not sure this is quite right
+    y = cython.operator.comma(1., t[0])
+    out(y, typeof(y))
+    del t
+
 def test_cmp():
     """
     >>> test_cmp()
@@ -148,6 +293,7 @@
     out(t[0] < 1, typeof(t[0] < 1))
     del t
 
+
 def test_index_call():
     """
     >>> test_index_call()
@@ -159,6 +305,20 @@
     out(t[0](100), typeof(t[0](100)))
     del t
 
+
+def test_index_assignment():
+    """
+    >>> test_index_assignment()
+    0 [int &]
+    123 [int [&]]
+    """
+    cdef RefTestOps* t = new RefTestOps()
+    iout(t[0][100], typeof(t[0][100]))
+    t[0][99] = 123
+    iout(t[0](100), typeof(t[0](100)))
+    del t
+
+
 def test_bool_op():
     """
     >>> test_bool_op()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_scoped_enums.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_scoped_enums.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_scoped_enums.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_scoped_enums.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,136 @@
+# mode: run
+# tag: cpp, cpp11
+
+from libcpp.limits cimport numeric_limits
+
+cdef extern from *:
+    """
+    enum class Enum1 {
+        Item1,
+        Item2
+    };
+    """
+    cdef enum class Enum1:
+        Item1
+        Item2
+
+
+cdef extern from * namespace "Namespace1":
+    """
+    namespace Namespace1 {
+        enum class Enum2 {
+            Item1,
+            Item2
+        };
+    }
+    """
+    cdef enum class Enum2:
+        Item1
+        Item2
+
+
+cdef enum class Enum3(int):
+    a = 1
+    b = 2
+
+
+cdef extern from *:
+    """
+    enum class sorted
+    {
+        a = 1,
+        b = 0
+    };
+    """
+    cdef enum class Enum4 "sorted":
+        a
+        b
+
+
+cdef extern from *:
+    """
+    #include 
+
+    enum class LongIntEnum : long int {
+        val = std::numeric_limits::max(),
+    };
+    """
+    enum class LongIntEnum(long int):
+        val
+
+
+def test_compare_enums():
+    """
+    >>> test_compare_enums()
+    (True, True, False, False)
+    """
+    cdef Enum1 x, y
+    x = Enum1.Item1
+    y = Enum1.Item2
+
+    return (
+        x == Enum1.Item1,
+        y == Enum1.Item2,
+        x == Enum1.Item2,
+        y == Enum1.Item1
+    )
+
+        
+def test_compare_namespace_enums():
+    """
+    >>> test_compare_enums()
+    (True, True, False, False)
+    """
+    cdef Enum2 z, w
+    
+    z = Enum2.Item1
+    w = Enum2.Item2
+
+    return (
+        z == Enum2.Item1,
+        w == Enum2.Item2,
+        z == Enum2.Item2,
+        w == Enum2.Item1
+    )
+
+
+def test_coerce_to_from_py_value(object i):
+    """
+    >>> test_coerce_to_from_py_value(1)
+    (True, False)
+
+    >>> test_coerce_to_from_py_value(2)
+    (False, True)
+
+    >>> test_coerce_to_from_py_value(3)
+    (False, False)
+
+    >>> test_coerce_to_from_py_value(11111111111111111111111111111111111111111111)
+    Traceback (most recent call last):
+    OverflowError: Python int too large to convert to C long
+    """
+    cdef Enum3 x = i
+    y = Enum3.b
+
+    return (
+        x == Enum3.a,
+        y == int(i)
+    )
+
+
+def test_reserved_cname():
+    """
+    >>> test_reserved_cname()
+    True
+    """
+    cdef Enum4 x = Enum4.a
+    return Enum4.a == int(1)
+
+
+def test_large_enum():
+    """
+    >>> test_large_enum()
+    True
+    """
+    long_max = int(numeric_limits[long].max())
+    return LongIntEnum.val == long_max
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_smart_ptr_helper.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_smart_ptr_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_smart_ptr_helper.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_smart_ptr_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -14,7 +14,7 @@
 
 template
 struct FreePtr {
-  void operator()( T * t ) noexcept
+  void operator()( T * t )
   {
     if(t != nullptr) {
       delete t;
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_smart_ptr.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_smart_ptr.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_smart_ptr.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_smart_ptr.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,8 +1,7 @@
-# distutils: extra_compile_args=-std=c++0x
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, cpp11, no-cpp-locals
 
-from libcpp.memory cimport unique_ptr, shared_ptr, default_delete 
+from libcpp.memory cimport unique_ptr, shared_ptr, default_delete, dynamic_pointer_cast
 from libcpp cimport nullptr
 
 cdef extern from "cpp_smart_ptr_helper.h":
@@ -12,6 +11,10 @@
     cdef cppclass FreePtr[T]:
         pass
 
+
+ctypedef const CountAllocDealloc const_CountAllocDealloc
+
+
 def test_unique_ptr():
     """
     >>> test_unique_ptr()
@@ -33,7 +36,7 @@
     x_ptr2.reset()
     assert alloc_count == 1
     assert dealloc_count == 1
-    
+
     alloc_count = 0
     dealloc_count = 0
     cdef unique_ptr[CountAllocDealloc,FreePtr[CountAllocDealloc]] x_ptr3
@@ -41,3 +44,60 @@
     assert x_ptr3.get() != nullptr;
     x_ptr3.reset()
     assert x_ptr3.get() == nullptr;
+
+
+def test_const_shared_ptr():
+    """
+    >>> test_const_shared_ptr()
+    """
+    cdef int alloc_count = 0, dealloc_count = 0
+    cdef shared_ptr[const CountAllocDealloc] ptr = shared_ptr[const_CountAllocDealloc](
+        new CountAllocDealloc(&alloc_count, &dealloc_count))
+    assert alloc_count == 1
+    assert dealloc_count == 0
+
+    cdef shared_ptr[const CountAllocDealloc] ptr2 = ptr
+    assert alloc_count == 1
+    assert dealloc_count == 0
+
+    ptr.reset()
+    assert alloc_count == 1
+    assert dealloc_count == 0
+
+    ptr2.reset()
+    assert alloc_count == 1
+    assert dealloc_count == 1
+
+
+cdef cppclass A:
+    void some_method():  # Force this to be a polymorphic class for dynamic cast.
+        pass
+
+cdef cppclass B(A):
+    pass
+
+cdef cppclass C(B):
+    pass
+
+cdef shared_ptr[A] holding_subclass = shared_ptr[A](new C())
+
+
+def test_assignment_to_base_class():
+    """
+    >>> test_assignment_to_base_class()
+    """
+    cdef shared_ptr[C] derived = shared_ptr[C](new C())
+    cdef shared_ptr[A] base = derived
+
+
+def test_dynamic_pointer_cast():
+    """
+    >>> test_dynamic_pointer_cast()
+    """
+    cdef shared_ptr[B] b = shared_ptr[B](new B())
+    cdef shared_ptr[A] a = dynamic_pointer_cast[A, B](b)
+    assert a.get() == b.get()
+
+    a = shared_ptr[A](new A())
+    b = dynamic_pointer_cast[B, A](a)
+    assert b.get() == NULL
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_static_method_overload.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_static_method_overload.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_static_method_overload.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_static_method_overload.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,49 @@
+# mode: run
+# tag: cpp, no-cpp-locals
+
+cdef extern from *:
+    """
+    struct Foo
+    {
+
+      static const char* bar(int x, int y) {
+        return "second";
+      }
+
+      static const char* bar(int x) {
+        return "first";
+      }
+
+      const char* baz(int x, int y) {
+        return "second";
+      }
+
+      const char* baz(int x) {
+        return "first";
+      }
+    };
+    """
+    cppclass Foo:
+        @staticmethod
+        const char* bar(int x)
+
+        @staticmethod
+        const char* bar(int x, int y)
+
+        const char* baz(int x)
+        const char* baz(int x, int y)
+
+def test_normal_method_overload():
+    """
+    >>> test_normal_method_overload()
+    """
+    cdef Foo f
+    assert f.baz(1) == b"first"
+    assert f.baz(1, 2) == b"second"
+
+def test_static_method_overload():
+    """
+    >>> test_static_method_overload()
+    """
+    assert Foo.bar(1) == b"first"
+    assert Foo.bar(1, 2) == b"second"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_comparison_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_comparison_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_comparison_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_comparison_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,67 @@
+# mode: run
+# tag: cpp, werror, cpp17, no-cpp-locals
+
+from libcpp cimport bool
+from libcpp.algorithm cimport equal, lexicographical_compare
+from libcpp.vector cimport vector
+
+cdef bool compare(int a, int b):
+    return a == b
+
+cdef bool less_than(char a, char b):
+    return a < b
+
+def test_equal(vector[int] v1, vector[int] v2):
+    """
+    Test equal.
+
+    >>> test_equal([1, 2, 3, 4], [1, 2, 3, 4])
+    True
+    >>> test_equal([1, 2, 3, 4], [9, 2, 3, 4])
+    False
+    """
+    return equal(v1.begin(), v1.end(), v2.begin())
+
+def test_equal_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test equal with binary predicate.
+
+    >>> test_equal_with_bin_pred([1, 2, 3, 4], [1, 2, 3, 4])
+    True
+    >>> test_equal_with_bin_pred([1, 2, 3, 4], [9, 2, 3, 4])
+    False
+    """
+    return equal(v1.begin(), v1.end(), v2.begin(), compare)
+
+def test_equal_with_second_range_and_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test equal with second range and binary predicate.
+
+    >>> test_equal_with_second_range_and_bin_pred([1, 2, 3, 4], [1, 2, 3, 4])
+    True
+    >>> test_equal_with_second_range_and_bin_pred([1, 2, 3, 4], [9, 2, 3, 4])
+    False
+    """
+    return equal(v1.begin(), v1.end(), v2.begin(), v2.end(), compare)
+
+def test_lexicographical_compare(vector[int] v1, vector[int] v2):
+    """
+    Test lexicographical_compare.
+
+    >>> test_lexicographical_compare([1, 2, 3, 4], [5, 6, 7, 8])
+    True
+    >>> test_lexicographical_compare([1, 2, 3, 4], [1, 1, 3, 4])
+    False
+    """
+    return lexicographical_compare(v1.begin(), v1.end(), v2.begin(), v2.end())
+
+def test_lexicographical_compare_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test lexicographical_compare with binary predicate
+
+    >>> test_lexicographical_compare_with_bin_pred([1, 2, 3, 4], [5, 6, 7, 8])
+    True
+    >>> test_lexicographical_compare_with_bin_pred([1, 2, 3, 4], [1, 1, 3, 4])
+    False
+    """
+    return lexicographical_compare(v1.begin(), v1.end(), v2.begin(), v2.end(), less_than)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_execpolicies.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_execpolicies.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_execpolicies.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_execpolicies.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,53 @@
+# mode: run
+# tag: cpp, werror, cpp17, cppexecpolicies, no-cpp-locals
+
+from libcpp.algorithm cimport is_sorted, sort, stable_sort, nth_element, all_of, count, copy
+from libcpp.execution cimport seq
+from libcpp.vector cimport vector
+from libcpp.functional cimport greater
+from libcpp.iterator cimport back_inserter
+from libcpp cimport bool
+
+
+def is_sorted_ints(vector[int] values):
+    """
+    Test is_sorted.
+
+    >>> is_sorted_ints([3, 1, 4, 1, 5])
+    False
+    >>> is_sorted_ints([1, 1, 3, 4, 5])
+    True
+    """
+    return is_sorted(seq, values.begin(), values.end())
+
+
+def sort_ints_reverse(vector[int] values):
+    """Test sort using a standard library comparison function object.
+
+    >>> sort_ints_reverse([5, 7, 4, 2, 8, 6, 1, 9, 0, 3])
+    [9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
+    """
+    sort(seq, values.begin(), values.end(), greater[int]())
+    return values
+
+
+def count_ones(vector[int] values):
+    """
+    Test count.
+
+    >>> count_ones([1, 2, 1, 2])
+    2
+    """
+    return count(seq, values.begin(), values.end(), 1)
+
+
+def copy_int(vector[int] values):
+    """
+    Test copy.
+
+    >>> copy_int(range(5))
+    [0, 1, 2, 3, 4]
+    """
+    cdef vector[int] out
+    copy(seq, values.begin(), values.end(), back_inserter(out))
+    return out
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_minmax_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_minmax_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_minmax_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_minmax_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,143 @@
+# mode: run
+# tag: cpp, werror, cpp17, cppexecpolicies
+
+from cython.operator cimport dereference as deref
+
+from libcpp cimport bool
+from libcpp.algorithm cimport (min_element, max_element, minmax, minmax_element, 
+                               clamp)
+from libcpp.vector cimport vector
+from libcpp.pair cimport pair
+from libcpp.execution cimport seq
+
+
+cdef bool less(int a, int b):
+    return a < b
+
+def test_min_element(vector[int] v):
+    """
+    Test min_element.
+
+    >>> test_min_element([0, 1, 2, 3, 4, 5])
+    0
+    """
+    cdef vector[int].iterator it = min_element(v.begin(), v.end())
+    return deref(it)
+
+def test_min_element_with_pred(vector[int] v):
+    """
+    Test min_element with binary predicate.
+
+    >>> test_min_element_with_pred([0, 1, 2, 3, 4, 5])
+    0
+    """
+    cdef vector[int].iterator it = min_element(v.begin(), v.end(), less)
+    return deref(it)
+
+def test_min_element_with_exec(vector[int] v):
+    """
+    Test min_element with execution policy.
+
+    >>> test_min_element_with_exec([0, 1, 2, 3, 4, 5])
+    0
+    """
+    cdef vector[int].iterator it = min_element(seq, v.begin(), v.end())
+    return deref(it)
+
+def test_max_element(vector[int] v):
+    """
+    Test max_element.
+
+    >>> test_max_element([0, 1, 2, 3, 4, 5])
+    5
+    """
+    cdef vector[int].iterator it = max_element(v.begin(), v.end())
+    return deref(it)
+
+def test_max_element_with_pred(vector[int] v):
+    """
+    Test max_element with binary predicate.
+
+    >>> test_max_element_with_pred([0, 1, 2, 3, 4, 5])
+    5
+    """
+    cdef vector[int].iterator it = max_element(v.begin(), v.end(), less)
+    return deref(it)
+
+def test_max_element_with_exec(vector[int] v):
+    """
+    Test max_element with execution policy.
+
+    >>> test_max_element_with_exec([0, 1, 2, 3, 4, 5])
+    5
+    """
+    cdef vector[int].iterator it = max_element(seq, v.begin(), v.end())
+    return deref(it)
+
+def test_minmax(int a, int b):
+    """
+    Test minmax.
+
+    >>> test_minmax(10, 20)
+    [10, 20]
+    """
+    cdef pair[int, int] p = minmax(a, b)
+    return [p.first, p.second]
+
+def test_minmax_with_pred(int a, int b):
+    """
+    Test minmax with binary predicate.
+
+    >>> test_minmax_with_pred(10, 20)
+    [10, 20]
+    """
+    cdef pair[int, int] p = minmax(a, b, less)
+    return [p.first, p.second]
+
+def test_minmax_element(vector[int] v):
+    """
+    Test minmax_element.
+
+    >>> test_minmax_element([0, 1, 2, 3, 4, 5])
+    [0, 5]
+    """
+    cdef pair[vector[int].iterator, vector[int].iterator] p = minmax_element(v.begin(), v.end())
+    return [deref(p.first), deref(p.second)]
+
+def test_minmax_element_with_pred(vector[int] v):
+    """
+    Test minmax_element with binary predicate.
+
+    >>> test_minmax_element_with_pred([0, 1, 2, 3, 4, 5])
+    [0, 5]
+    """
+    cdef pair[vector[int].iterator, vector[int].iterator] p = minmax_element(v.begin(), v.end(), less)
+    return [deref(p.first), deref(p.second)]
+
+def test_minmax_element_with_exec(vector[int] v):
+    """
+    Test minmax_element with execution policy.
+
+    >>> test_minmax_element_with_exec([0, 1, 2, 3, 4, 5])
+    [0, 5]
+    """
+    cdef pair[vector[int].iterator, vector[int].iterator] p = minmax_element(seq, v.begin(), v.end())
+    return [deref(p.first), deref(p.second)]
+
+def test_clamp(int v, int lo, int hi):
+    """
+    Test clamp.
+
+    >>> test_clamp(-129, -128, 255)
+    -128
+    """
+    return clamp(v, lo, hi)
+
+def test_clamp_with_pred(int v, int lo, int hi):
+    """
+    Test clamp with binary predicate
+
+    >>> test_clamp_with_pred(-129, -128, 255)
+    -128
+    """
+    return clamp(v, lo, hi, less)
\ No newline at end of file
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,436 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+from __future__ import print_function
+
+from cython.operator cimport dereference as deref
+from cython.operator cimport preincrement, postincrement
+from libcpp cimport bool
+from libcpp.algorithm cimport copy, copy_if, copy_n, copy_backward, move, move_backward, fill, fill_n, transform
+from libcpp.algorithm cimport generate, generate_n, remove, remove_if, remove_copy, remove_copy_if, replace, replace_if
+from libcpp.algorithm cimport replace_copy, replace_copy_if, swap, swap_ranges, iter_swap, reverse, reverse_copy
+from libcpp.algorithm cimport rotate, rotate_copy, unique, unique_copy
+from libcpp.algorithm cimport sort, upper_bound, min_element, max_element
+from libcpp.iterator cimport back_inserter
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+
+def copy_int(vector[int] values):
+    """
+    Test copy.
+
+    >>> copy_int(range(5))
+    [0, 1, 2, 3, 4]
+    """
+    cdef vector[int] out
+    copy(values.begin(), values.end(), back_inserter(out))
+    return out
+
+
+cdef bool is_odd(int i):
+    return i % 2
+
+
+def copy_int_if_odd(vector[int] values):
+    """
+    Test copy_if.
+
+    >>> copy_int_if_odd(range(5))
+    [1, 3]
+    """
+    cdef vector[int] out
+    copy_if(values.begin(), values.end(), back_inserter(out), is_odd)
+    return out
+
+
+def copy_int_n(vector[int] values, int count):
+    """
+    Test copy_n.
+
+    >>> copy_int_n(range(5), 2)
+    [0, 1]
+    """
+    cdef vector[int] out
+    copy_n(values.begin(), count, back_inserter(out))
+    return out
+
+
+def copy_int_backward(vector[int] values):
+    """
+    Test copy_backward.
+
+    >>> copy_int_backward(range(5))
+    [0, 0, 0, 0, 1, 2, 3, 4]
+    """
+    out = vector[int](values.size() + 3)
+    copy_backward(values.begin(), values.end(), out.end())
+    return out
+
+
+def move_int(vector[int] values):
+    """
+    Test move.
+
+    >>> move_int(range(5))
+    [0, 1, 2, 3, 4]
+    """
+    cdef vector[int] out
+    move(values.begin(), values.end(), back_inserter(out))
+    return out
+
+
+def move_int_backward(vector[int] values):
+    """
+    Test move_backward.
+
+    >>> move_int_backward(range(5))
+    [0, 0, 0, 0, 1, 2, 3, 4]
+    """
+    out = vector[int](values.size() + 3)
+    move_backward(values.begin(), values.end(), out.end())
+    return out
+
+
+def fill_int(vector[int] array, int value):
+    """
+    Test fill.
+
+    >>> fill_int(range(5), -1)
+    [-1, -1, -1, -1, -1]
+    """
+    fill(array.begin(), array.end(), value)
+    return array
+
+
+def fill_int_n(vector[int] array, int count, int value):
+    """
+    Test fill_n.
+
+    >>> fill_int_n(range(5), 3, -1)
+    [-1, -1, -1, 3, 4]
+    """
+    fill_n(array.begin(), count, value)
+    return array
+
+
+cdef int to_ord(unsigned char c):
+    return c
+
+
+def string_to_ord(string s):
+    """
+    Test transform (unary version).
+
+    >> string_to_ord(b"HELLO")
+    [72, 69, 76, 76, 79]
+    """
+    cdef vector[int] ordinals
+    transform(s.begin(), s.end(), back_inserter(ordinals), to_ord)
+    return ordinals
+
+
+cdef int add_ints(int lhs, int rhs):
+    return lhs + rhs
+
+
+def add_int_vectors(vector[int] lhs, vector[int] rhs):
+    """
+    Test transform (binary version).
+
+    >>> add_int_vectors([1, 2, 3], [4, 5, 6])
+    [5, 7, 9]
+    """
+    transform(lhs.begin(), lhs.end(), rhs.begin(), lhs.begin(), add_ints)
+    return lhs
+
+
+cdef int i = 0
+cdef int generator():
+    return postincrement(i)
+
+
+def generate_ints(int count):
+    """
+    Test generate.
+
+    >> generate_ints(5)
+    [0, 1, 2, 3, 4]
+    """
+    out = vector[int](count)
+    generate(out.begin(), out.end(), generator)
+    return out
+
+
+cdef int j = 0
+cdef int generator2():
+    return postincrement(j)
+
+
+def generate_n_ints(int count):
+    """
+    Test generate_n.
+
+    >> generate_n_ints(5)
+    [0, 1, 2, 3, 4, 0, 0, 0]
+    """
+    out = vector[int](count + 3)
+    generate_n(out.begin(), count, generator2)
+    return out
+
+
+def remove_spaces(string s):
+    """
+    Test remove.
+
+    >>> print(remove_spaces(b"Text with some   spaces").decode("ascii"))
+    Textwithsomespaces
+    """
+    s.erase(remove(s.begin(), s.end(), ord(" ")), s.end())
+    return s
+
+
+cdef bool is_whitespace(unsigned char c) except -1:
+    # std::isspace from 
+    return chr(c) in " \f\n\r\t\v"
+
+
+def remove_whitespace(string s):
+    r"""
+    Test remove_if.
+
+    >>> print(remove_whitespace(b"Text\n with\tsome \t  whitespaces\n\n").decode("ascii"))
+    Textwithsomewhitespaces
+    """
+    s.erase(remove_if(s.begin(), s.end(), &is_whitespace), s.end())
+    return s
+
+
+def remove_spaces2(string s):
+    """
+    Test remove_copy.
+
+    >>> print(remove_spaces2(b"Text with some   spaces").decode("ascii"))
+    Textwithsomespaces
+    """
+    cdef string out
+    remove_copy(s.begin(), s.end(), back_inserter(out), ord(" "))
+    return out
+
+
+def remove_whitespace2(string s):
+    r"""
+    Test remove_copy_if.
+
+    >>> print(remove_whitespace2(b"Text\n with\tsome \t  whitespaces\n\n").decode("ascii"))
+    Textwithsomewhitespaces
+    """
+    cdef string out
+    remove_copy_if(s.begin(), s.end(), back_inserter(out), &is_whitespace)
+    return out
+
+
+def replace_ints(vector[int] values, int old, int new):
+    """
+    Test replace.
+
+    >>> replace_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88)
+    [5, 7, 4, 2, 88, 6, 1, 9, 0, 3]
+    """
+    replace(values.begin(), values.end(), old, new)
+    return values
+
+
+cdef bool less_than_five(int i):
+    return i < 5
+
+
+def replace_ints_less_than_five(vector[int] values, int new):
+    """
+    Test replace_if (using cppreference example that doesn't translate well).
+
+    >>> replace_ints_less_than_five([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55)
+    [5, 7, 55, 55, 88, 6, 55, 9, 55, 55]
+    """
+    replace_if(values.begin(), values.end(), less_than_five, new)
+    return values
+
+
+def replace_ints2(vector[int] values, int old, int new):
+    """
+    Test replace_copy.
+
+    >>> replace_ints2([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88)
+    [5, 7, 4, 2, 88, 6, 1, 9, 0, 3]
+    """
+    cdef vector[int] out
+    replace_copy(values.begin(), values.end(), back_inserter(out), old, new)
+    return out
+
+
+def replace_ints_less_than_five2(vector[int] values, int new):
+    """
+    Test replace_copy_if (using cppreference example that doesn't translate well).
+
+    >>> replace_ints_less_than_five2([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55)
+    [5, 7, 55, 55, 88, 6, 55, 9, 55, 55]
+    """
+    cdef vector[int] out
+    replace_copy_if(values.begin(), values.end(), back_inserter(out), less_than_five, new)
+    return out
+
+
+def test_swap_ints():
+    """
+    >>> test_swap_ints()
+    5 3
+    3 5
+    """
+    cdef int a = 5, b = 3
+    print(a, b)
+    swap(a, b)
+    print(a, b)
+
+
+def test_swap_vectors():
+    """
+    >>> test_swap_vectors()
+    [1, 2, 3] [4, 5, 6]
+    [4, 5, 6] [1, 2, 3]
+    """
+    cdef vector[int] a = [1, 2, 3], b = [4, 5, 6]
+    print(a, b)
+    swap(a, b)
+    print(a, b)
+
+
+def test_swap_ranges():
+    """
+    >>> test_swap_ranges()
+    [1, 2, 3] [4, 5, 6]
+    [4, 5, 6] [1, 2, 3]
+    """
+    cdef vector[int] a = [1, 2, 3], b = [4, 5, 6]
+    print(a, b)
+    swap_ranges(a.begin(), a.end(), b.begin())
+    print(a, b)
+
+
+def selection_sort(vector[int] values, reversed=False):
+    """
+    Test iter_swap using cppreference example. Extra "reversed argument tests max_element
+
+    >>> selection_sort([-7, 6, 2, 4, -1, 6, -9, -1, 2, -5, 10, -9, -5, -3, -5, -3, 6, 6, 1, 8])
+    [-9, -9, -7, -5, -5, -5, -3, -3, -1, -1, 1, 2, 2, 4, 6, 6, 6, 6, 8, 10]
+    >>> selection_sort([-7, 6, 2, 4, -1, 6, -9, -1, 2, -5, 10, -9, -5, -3, -5, -3, 6, 6, 1, 8], reversed=True)
+    [10, 8, 6, 6, 6, 6, 4, 2, 2, 1, -1, -1, -3, -3, -5, -5, -5, -7, -9, -9]
+    """
+    i = values.begin()
+    end = values.end()
+    while i < end:
+        iter_swap(i, min_element(i, end) if not reversed else max_element(i,end))
+        preincrement(i)
+    return values
+
+
+def reverse_ints(vector[int] values):
+    """
+    Test reverse.
+
+    >>> reverse_ints([1, 2, 3])
+    [3, 2, 1]
+    """
+    reverse(values.begin(), values.end())
+    return values
+
+
+def reverse_ints2(vector[int] values):
+    """
+    Test reverse_copy.
+
+    >>> reverse_ints2([1, 2, 3])
+    [3, 2, 1]
+    """
+    cdef vector[int] out
+    reverse_copy(values.begin(), values.end(), back_inserter(out))
+    return out
+
+
+def insertion_sort(vector[int] values):
+    """
+    Test rotate using cppreference example.
+
+    >>> insertion_sort([2, 4, 2, 0, 5, 10, 7, 3, 7, 1])
+    [0, 1, 2, 2, 3, 4, 5, 7, 7, 10]
+    """
+    i = values.begin()
+    while i < values.end():
+        rotate(upper_bound(values.begin(), i, deref(i)), i, i + 1)
+        preincrement(i)
+    return values
+
+
+def rotate_ints_about_middle(vector[int] values):
+    """
+    Test rotate_copy.
+
+    >>> rotate_ints_about_middle([1, 2, 3, 4, 5])
+    [3, 4, 5, 1, 2]
+    """
+    cdef vector[int] out
+    cdef vector[int].iterator pivot = values.begin() + values.size()/2
+    rotate_copy(values.begin(), pivot, values.end(), back_inserter(out))
+    return out
+
+
+def unique_ints(vector[int] values):
+    """
+    Test unique.
+
+    >>> unique_ints([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7])
+    [1, 2, 3, 4, 5, 6, 7]
+    """
+    sort(values.begin(), values.end())
+    values.erase(unique(values.begin(), values.end()), values.end())
+    return values
+
+
+cdef bool both_space(unsigned char lhs, unsigned char rhs):
+    return lhs == rhs == ord(' ')
+
+
+def collapse_spaces(string text):
+    """
+    Test unique (predicate version) using cppreference example for unique_copy.
+
+    >>> print(collapse_spaces(b"The      string    with many       spaces!").decode("ascii"))
+    The string with many spaces!
+    """
+    last = unique(text.begin(), text.end(), &both_space)
+    text.erase(last, text.end())
+    return text
+
+
+def unique_ints2(vector[int] values):
+    """
+    Test unique_copy.
+
+    >>> unique_ints2([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7])
+    [1, 2, 3, 4, 5, 6, 7]
+    """
+    cdef vector[int] out
+    sort(values.begin(), values.end())
+    unique_copy(values.begin(), values.end(), back_inserter(out))
+    return out
+
+
+def collapse_spaces2(string text):
+    """
+    Test unique_copy (predicate version) using cppreference example.
+
+    >>> print(collapse_spaces2(b"The      string    with many       spaces!").decode("ascii"))
+    The string with many spaces!
+    """
+    cdef string out
+    unique_copy(text.begin(), text.end(), back_inserter(out), &both_space)
+    return out
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,307 @@
+# mode: run
+# tag: cpp, werror, cpp11
+
+from cython.operator cimport dereference as deref
+
+from libcpp cimport bool
+from libcpp.algorithm cimport all_of, any_of, none_of, for_each, count, count_if, mismatch, find, find_if, find_if_not
+from libcpp.algorithm cimport find_end, find_first_of, adjacent_find, search, search_n
+from libcpp.iterator cimport distance
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+
+cdef bool is_odd(int i):
+    return i % 2
+
+
+def all_odd(vector[int] values):
+    """
+    Test all_of with is_odd predicate.
+
+    >>> all_odd([3, 5, 7, 11, 13, 17, 19, 23])
+    True
+    >>> all_odd([3, 4])
+    False
+    """
+    return all_of(values.begin(), values.end(), is_odd)
+
+
+def any_odd(vector[int] values):
+    """
+    Test any_of with is_odd predicate.
+
+    >>> any_odd([1, 2, 3, 4])
+    True
+    >>> any_odd([2, 4, 6, 8])
+    False
+    """
+    return any_of(values.begin(), values.end(), is_odd)
+
+
+def none_odd(vector[int] values):
+    """
+    Test none_of with is_odd predicate.
+
+    >>> none_odd([2, 4, 6, 8])
+    True
+    >>> none_odd([1, 2, 3, 4])
+    False
+    """
+    return none_of(values.begin(), values.end(), is_odd)
+
+
+def count_ones(vector[int] values):
+    """
+    Test count.
+
+    >>> count_ones([1, 2, 1, 2])
+    2
+    """
+    return count(values.begin(), values.end(), 1)
+
+
+cdef void add_one(int &i):
+    # https://github.com/cython/cython/issues/1863
+    (&i)[0] += 1
+
+
+def increment_ints(vector[int] values):
+    """
+    Test for_each.
+
+    >>> increment_ints([3, 4, 2, 8, 15, 267])
+    [4, 5, 3, 9, 16, 268]
+    """
+    for_each(values.begin(), values.end(), &add_one)
+    return values
+
+
+def count_odd(vector[int] values):
+    """
+    Test count_if with is_odd predicate.
+
+    >>> count_odd([1, 2, 3, 4])
+    2
+    >>> count_odd([2, 4, 6, 8])
+    0
+    """
+    return count_if(values.begin(), values.end(), is_odd)
+
+
+def mirror_ends(string data):
+    """
+    Test mismatch using cppreference example.
+
+    This program determines the longest substring that is simultaneously found at the very beginning of the given string
+    and at the very end of it, in reverse order (possibly overlapping).
+
+    >>> print(mirror_ends(b'abXYZba').decode('ascii'))
+    ab
+    >>> print(mirror_ends(b'abca').decode('ascii'))
+    a
+    >>> print(mirror_ends(b'aba').decode('ascii'))
+    aba
+    """
+    return string(data.begin(), mismatch(data.begin(), data.end(), data.rbegin()).first)
+
+
+def mismatch_ints(vector[int] values1, vector[int] values2):
+    """
+    Test mismatch(first1, last1, first2).
+
+    >>> mismatch_ints([1, 2, 3], [1, 2, 3])
+    >>> mismatch_ints([1, 2], [1, 2, 3])
+    >>> mismatch_ints([1, 3], [1, 2, 3])
+    (3, 2)
+    """
+    result = mismatch(values1.begin(), values1.end(), values2.begin())
+    if result.first == values1.end():
+        return
+    return deref(result.first), deref(result.second)
+
+
+def is_int_in(vector[int] values, int target):
+    """
+    Test find.
+
+    >>> is_int_in(range(5), 3)
+    True
+    >>> is_int_in(range(5), 10)
+    False
+    """
+    return find(values.begin(), values.end(), target) != values.end()
+
+
+def find_odd(vector[int] values):
+    """
+    Test find_if using is_odd predicate.
+
+    >>> find_odd([2, 3, 4])
+    3
+    >>> find_odd([2, 4, 6])
+    """
+    result = find_if(values.begin(), values.end(), is_odd)
+    if result != values.end():
+        return deref(result)
+    else:
+        return None
+
+
+def find_even(vector[int] values):
+    """
+    Test find_if_not using is_odd predicate.
+
+    >>> find_even([3, 4, 5])
+    4
+    >>> find_even([1, 3, 5])
+    """
+    result = find_if_not(values.begin(), values.end(), is_odd)
+    if result != values.end():
+        return deref(result)
+    else:
+        return None
+
+
+def find_last_int_sequence(vector[int] values, vector[int] target):
+    """
+    Test find_end.
+
+    >>> find_last_int_sequence([1, 2, 3, 1, 2, 3], [2, 3])
+    4
+    >>> find_last_int_sequence([1, 2, 3], [4, 5])
+    """
+    result = find_end(values.begin(), values.end(), target.begin(), target.end())
+    if result != values.end():
+        return distance(values.begin(), result)
+    else:
+        return None
+
+
+cdef bool is_equal(int lhs, int rhs):
+    return lhs == rhs
+
+
+def find_last_int_sequence2(vector[int] values, vector[int] target):
+    """
+    Test find_end (using is_equal predicate).
+
+    >>> find_last_int_sequence2([1, 2, 3, 1, 2, 3], [2, 3])
+    4
+    >>> find_last_int_sequence2([1, 2, 3], [4, 5])
+    """
+    result = find_end(values.begin(), values.end(), target.begin(), target.end(), &is_equal)
+    if result != values.end():
+        return distance(values.begin(), result)
+    else:
+        return None
+
+
+def find_first_int_in_set(values, target):
+    """
+    Test find_first_of.
+
+    >>> find_first_int_in_set([1, 2, 3, 4, 5], [3, 5])
+    2
+    >>> find_first_int_in_set([1, 2, 3], [4, 5])
+    """
+    cdef vector[int] v = values
+    cdef vector[int] t = target
+    result = find_first_of(v.begin(), v.end(), t.begin(), t.end())
+    if result != v.end():
+        return distance(v.begin(), result)
+    else:
+        return None
+
+
+def find_first_int_in_set2(vector[int] values, vector[int] target):
+    """
+    Test find_first_of with is_equal predicate.
+
+    >>> find_first_int_in_set2([1, 2, 3, 4, 5], [3, 5])
+    2
+    >>> find_first_int_in_set2([1, 2, 3], [4, 5])
+    """
+    result = find_first_of(values.begin(), values.end(), target.begin(), target.end(), is_equal)
+    if result != values.end():
+        return distance(values.begin(), result)
+    else:
+        return None
+
+
+def find_adjacent_int(vector[int] values):
+    """
+    Test adjacent_find.
+
+    >>> find_adjacent_int([0, 1, 2, 3, 40, 40, 41, 41, 5])
+    4
+    >>> find_adjacent_int(range(5))
+    """
+    result = adjacent_find(values.begin(), values.end())
+    if result != values.end():
+        return distance(values.begin(), result)
+    else:
+        return None
+
+
+def find_adjacent_int2(vector[int] values):
+    """
+    Test find_adjacent with is_equal predicate.
+
+    >>> find_adjacent_int2([0, 1, 2, 3, 40, 40, 41, 41, 5])
+    4
+    >>> find_adjacent_int2(range(5))
+    """
+    result = adjacent_find(values.begin(), values.end(), is_equal)
+    if result != values.end():
+        return distance(values.begin(), result)
+    else:
+        return None
+
+
+def in_quote(string quote, string word):
+    """
+    Test search using cppreference example.
+
+    >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"learning")
+    True
+    >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"lemming")
+    False
+    """
+    return search(quote.begin(), quote.end(), word.begin(), word.end()) != quote.end()
+
+
+def in_quote2(string quote, string word):
+    """
+    Test search using cppreference example (with is_equal predicate).
+
+    >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"learning")
+    True
+    >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"lemming")
+    False
+    """
+    return search(quote.begin(), quote.end(), word.begin(), word.end(), &is_equal) != quote.end()
+
+
+def consecutive_values(string c, int count, char v):
+    """
+    Test search_n using cppreference example (without std::begin and std::end).
+
+    >>> consecutive_values(b"1001010100010101001010101", 4, ord("0"))
+    False
+    >>> consecutive_values(b"1001010100010101001010101", 3, ord("0"))
+    True
+    """
+    return search_n(c.begin(), c.end(), count, v) != c.end()
+
+
+def consecutive_values2(string c, int count, char v):
+    """
+    Test search_n using cppreference example (with is_equal predicate).
+
+    >>> consecutive_values2(b"1001010100010101001010101", 4, ord("0"))
+    False
+    >>> consecutive_values2(b"1001010100010101001010101", 3, ord("0"))
+    True
+    """
+    return search_n(c.begin(), c.end(), count, v, &is_equal) != c.end()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_partitioning_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_partitioning_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_partitioning_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_partitioning_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,90 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+from __future__ import print_function
+
+from libcpp cimport bool
+from libcpp.algorithm cimport is_partitioned, partition, partition_copy, stable_partition, partition_point
+from libcpp.algorithm cimport for_each, copy, reverse
+from libcpp.iterator cimport back_inserter
+from libcpp.vector cimport vector
+
+
+cdef bool is_even(int i):
+    return i % 2 == 0
+
+
+def test_is_partitioned():
+    """
+    >>> test_is_partitioned()
+    False
+    True
+    False
+    """
+    cdef vector[int] values = range(10)
+    print(is_partitioned(values.begin(), values.end(), is_even))
+
+    partition(values.begin(), values.end(), &is_even)
+    print(is_partitioned(values.begin(), values.end(), is_even))
+
+    reverse(values.begin(), values.end())
+    print(is_partitioned(values.begin(), values.end(), is_even))
+
+
+cdef int print_int(int v) except -1:
+    print(v, end=" ")
+
+
+def print_partition(vector[int] values):
+    """
+    Test partition.
+
+    >> print_partition(range(10))
+    0 8 2 6 4  *  5 3 7 1 9
+    """
+    it = partition(values.begin(), values.end(), &is_even)
+    for_each(values.begin(), it, &print_int)
+    print("*", end=" ")
+    for_each(it, values.end(), &print_int)
+    print()
+
+
+def partition_ints_even(vector[int] values):
+    """
+    Test partition_copy.
+
+    >>> partition_ints_even(range(10))
+    ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9])
+    """
+    cdef vector[int] even_values, odd_values
+    partition_copy(values.begin(), values.end(), back_inserter(even_values), back_inserter(odd_values), &is_even)
+    return even_values, odd_values
+
+
+cdef bool is_positive(int v):
+    return v > 0
+
+
+def partition_ints_positive(vector[int] values):
+    """
+    Test stable_partition.
+
+    >>> partition_ints_positive([0, 0, 3, 0, 2, 4, 5, 0, 7])
+    [3, 2, 4, 5, 7, 0, 0, 0, 0]
+    """
+    stable_partition(values.begin(), values.end(), &is_positive)
+    return values
+
+
+def partition_point_ints_even(vector[int] values):
+    """
+    Test partition_point.
+
+    >>> partition_point_ints_even([0, 8, 2, 6, 4, 5, 3, 7, 1, 9])
+    ([0, 8, 2, 6, 4], [5, 3, 7, 1, 9])
+    """
+    it = partition_point(values.begin(), values.end(), is_even)
+    cdef vector[int] even_values, odd_values
+    copy(values.begin(), it, back_inserter(even_values))
+    copy(it, values.end(), back_inserter(odd_values))
+    return even_values, odd_values
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_permutation_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_permutation_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_permutation_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_permutation_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,103 @@
+# mode: run
+# tag: cpp, werror, cpp17, no-cpp-locals, c_string_type=str
+# cython: c_string_encoding=ascii, c_string_type=str
+
+from libcpp cimport bool
+from libcpp.algorithm cimport is_permutation, next_permutation, prev_permutation
+from libcpp.vector cimport vector
+from libcpp.string cimport string
+
+cdef bool compare(int a, int b):
+    return a == b
+
+cdef bool less_than(char a, char b):
+    return a < b
+
+def test_is_permutation(vector[int] v1, vector[int] v2):
+    """
+    Test is_permutation.
+
+    >>> test_is_permutation([1, 2, 3, 4], [4, 2, 3, 1])
+    True
+    >>> test_is_permutation([1, 2, 3, 4], [4, 4, 2, 5])
+    False
+    """
+    return is_permutation(v1.begin(), v1.end(), v2.begin())
+
+def test_is_permutation_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test is_permutation with binary predicate
+
+    >>> test_is_permutation_with_bin_pred([1, 2, 3, 4], [4, 2, 3, 1])
+    True
+    >>> test_is_permutation_with_bin_pred([1, 2, 3, 4], [4, 4, 2, 5])
+    False
+    """
+    return is_permutation(v1.begin(), v1.end(), v2.begin(), compare)
+
+def test_is_permutation_with_second_range_and_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test is_permutation with second range and binary predicate
+
+    >>> test_is_permutation_with_second_range_and_bin_pred([1, 2, 3, 4], [4, 2, 3, 1])
+    True
+    >>> test_is_permutation_with_second_range_and_bin_pred([1, 2, 3, 4], [4, 4, 2, 5])
+    False
+    """
+    return is_permutation(v1.begin(), v1.end(), v2.begin(), v2.end(), compare)
+
+def test_next_permutation(s_in, s_perm):
+    """
+    Test next_permutation.
+
+    >>> test_next_permutation("aba", "baa")
+    True
+    >>> test_next_permutation("aba", "bab")
+    False
+    """
+    cdef string ss = s_in
+    cdef string expected = s_perm
+    next_permutation(ss.begin(), ss.end())
+    return ss == expected
+
+def test_next_permutation_with_bin_pred(s_in, s_perm):
+    """
+    Test next_permutation with binary predicate
+
+    >>> test_next_permutation_with_bin_pred("aba", "baa")
+    True
+    >>> test_next_permutation_with_bin_pred("aba", "bab")
+    False
+    """
+    cdef string ss = s_in
+    cdef string expected = s_perm
+    next_permutation(ss.begin(), ss.end(), less_than)
+    return ss == expected
+
+def test_prev_permutation(s_in, s_perm):
+    """
+    Test prev_permutation.
+
+    >>> test_prev_permutation("aba", "aab")
+    True
+    >>> test_prev_permutation("aba", "bab")
+    False
+    """
+    cdef string ss = s_in
+    cdef string expected = s_perm
+    prev_permutation(ss.begin(), ss.end())
+    return ss == expected
+
+def test_prev_permutation_with_bin_pred(s_in, s_perm):
+    """
+    Test prev_permutation with binary predicate
+
+    >>> test_prev_permutation_with_bin_pred("aba", "aab")
+    True
+    >>> test_prev_permutation_with_bin_pred("aba", "bab")
+    False
+    """
+    cdef string ss = s_in
+    cdef string expected = s_perm
+    prev_permutation(ss.begin(), ss.end(), less_than)
+    return ss == expected
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorted_ranges_other_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorted_ranges_other_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorted_ranges_other_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorted_ranges_other_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,54 @@
+# mode: run
+# tag: cpp, werror, cpp11
+
+from cython.operator cimport dereference as deref
+
+from libcpp cimport bool
+from libcpp.algorithm cimport merge, inplace_merge
+from libcpp.vector cimport vector
+
+
+cdef bool less(int a, int b):
+    return a < b
+
+def test_merge(vector[int] v1, vector[int] v2):
+    """
+    Test merge.
+
+    >>> test_merge([1, 3, 5], [2, 4])
+    [1, 2, 3, 4, 5]
+    """
+    cdef vector[int] out = vector[int](v1.size() + v2.size())
+    merge(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin())
+    return out
+
+def test_merge_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test merge with binary predicate
+
+    >>> test_merge_with_bin_pred([1, 3, 5], [2, 4])
+    [1, 2, 3, 4, 5]
+    """
+    cdef vector[int] out = vector[int](v1.size() + v2.size())
+    merge(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin(), less)
+    return out
+
+def test_inplace_merge(vector[int] v):
+    """
+    Test inplace_merge.
+
+    >>> test_inplace_merge([4, 5, 6, 1, 2, 3])
+    [1, 2, 3, 4, 5, 6]
+    """
+    inplace_merge(v.begin(), v.begin() + 3, v.end())
+    return v
+
+def test_inplace_merge_with_bin_pred(vector[int] v):
+    """
+    Test inplace_merge with binary predicate
+
+    >>> test_inplace_merge_with_bin_pred([4, 5, 6, 1, 2, 3])
+    [1, 2, 3, 4, 5, 6]
+    """
+    inplace_merge(v.begin(), v.begin() + 3, v.end(), less)
+    return v
\ No newline at end of file
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorted_ranges_set_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorted_ranges_set_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorted_ranges_set_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorted_ranges_set_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,121 @@
+# mode: run
+# tag: cpp, werror, cpp11
+
+from libcpp cimport bool
+from libcpp.algorithm cimport (includes, set_difference, set_intersection, 
+                               set_symmetric_difference, set_union)
+from libcpp.vector cimport vector
+
+
+cdef bool less(int a, int b):
+    return a < b
+
+def test_includes(vector[int] v1, vector[int] v2):
+    """
+    Test includes.
+
+    >>> test_includes([1, 2, 3, 4], [1, 2, 3])
+    True
+    >>> test_includes([1, 2, 3, 4], [5, 6, 7])
+    False
+    """
+    return includes(v1.begin(), v1.end(), v2.begin(), v2.end())
+
+def test_includes_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test includes with binary predicate
+
+    >>> test_includes_with_bin_pred([1, 2, 3, 4], [1, 2, 3])
+    True
+    >>> test_includes_with_bin_pred([1, 2, 3, 4], [5, 6, 7])
+    False
+    """
+    return includes(v1.begin(), v1.end(), v2.begin(), v2.end(), less)
+
+def test_set_difference(vector[int] v1, vector[int] v2):
+    """
+    Test set_difference.
+
+    >>> test_set_difference([1, 2, 5, 5, 5, 9], [2, 5, 7])
+    [1, 5, 5, 9]
+    """
+    cdef vector[int] diff = vector[int](4)
+    set_difference(v1.begin(), v1.end(), v2.begin(), v2.end(), diff.begin())
+    return diff
+
+def test_set_difference_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test set_difference with binary predicate
+
+    >>> test_set_difference_with_bin_pred([1, 2, 5, 5, 5, 9], [2, 5, 7])
+    [1, 5, 5, 9]
+    """
+    cdef vector[int] diff = vector[int](4)
+    set_difference(v1.begin(), v1.end(), v2.begin(), v2.end(), diff.begin(), less)
+    return diff
+
+def test_set_intersection(vector[int] v1, vector[int] v2):
+    """
+    Test set_intersection.
+
+    >>> test_set_intersection([1, 2, 3, 4, 5, 6, 7, 8], [5, 7, 9, 10])
+    [5, 7]
+    """
+    cdef vector[int] out = vector[int](2)
+    set_intersection(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin())
+    return out
+
+def test_set_intersection_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test set_intersection with binary predicate
+
+    >>> test_set_intersection_with_bin_pred([1, 2, 3, 4, 5, 6, 7, 8], [5, 7, 9, 10])
+    [5, 7]
+    """
+    cdef vector[int] out = vector[int](2)
+    set_intersection(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin(), less)
+    return out
+
+def test_set_symmetric_difference(vector[int] v1, vector[int] v2):
+    """
+    Test set_symmetric_difference.
+
+    >>> test_set_symmetric_difference([1, 2, 3, 4, 5, 6, 7, 8], [5, 7, 9, 10])
+    [1, 2, 3, 4, 6, 8, 9, 10]
+    """
+    cdef vector[int] out = vector[int](8)
+    set_symmetric_difference(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin())
+    return out
+
+def test_set_symmetric_difference_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test set_symmetric_difference with binary predicate
+
+    >>> test_set_symmetric_difference_with_bin_pred([1, 2, 3, 4, 5, 6, 7, 8], [5, 7, 9, 10])
+    [1, 2, 3, 4, 6, 8, 9, 10]
+    """
+    cdef vector[int] out = vector[int](8)
+    set_symmetric_difference(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin(), less)
+    return out
+
+def test_set_union(vector[int] v1, vector[int] v2):
+    """
+    Test set_union.
+
+    >>> test_set_union([1, 2, 3, 4, 5], [3, 4, 5, 6, 7])
+    [1, 2, 3, 4, 5, 6, 7]
+    """
+    cdef vector[int] out = vector[int](7)
+    set_union(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin())
+    return out
+
+def test_set_union_with_bin_pred(vector[int] v1, vector[int] v2):
+    """
+    Test set_union with binary predicate
+
+    >>> test_set_union_with_bin_pred([1, 2, 3, 4, 5], [3, 4, 5, 6, 7])
+    [1, 2, 3, 4, 5, 6, 7]
+    """
+    cdef vector[int] out = vector[int](7)
+    set_union(v1.begin(), v1.end(), v2.begin(), v2.end(), out.begin(), less)
+    return out
\ No newline at end of file
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorting_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorting_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_algo_sorting_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_algo_sorting_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,187 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+from __future__ import print_function
+
+from libcpp cimport bool
+from libcpp.algorithm cimport is_sorted, is_sorted_until, sort, partial_sort, partial_sort_copy, stable_sort
+from libcpp.algorithm cimport nth_element
+from libcpp.functional cimport greater
+from libcpp.iterator cimport distance
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+
+
+def is_sorted_ints(vector[int] values):
+    """
+    Test is_sorted.
+
+    >>> is_sorted_ints([3, 1, 4, 1, 5])
+    False
+    >>> is_sorted_ints([1, 1, 3, 4, 5])
+    True
+    """
+    return is_sorted(values.begin(), values.end())
+
+
+def initial_sorted_elements(vector[int] values):
+    """
+    Test is_sorted_until.
+
+    >>> initial_sorted_elements([4, 1, 9, 5, 1, 3])
+    1
+    >>> initial_sorted_elements([4, 5, 9, 3, 1, 1])
+    3
+    >>> initial_sorted_elements([9, 3, 1, 4, 5, 1])
+    1
+    >>> initial_sorted_elements([1, 3, 5, 4, 1, 9])
+    3
+    >>> initial_sorted_elements([5, 9, 1, 1, 3, 4])
+    2
+    >>> initial_sorted_elements([4, 9, 1, 5, 1, 3])
+    2
+    >>> initial_sorted_elements([1, 1, 4, 9, 5, 3])
+    4
+    """
+    sorted_end = is_sorted_until(values.begin(), values.end())
+    return distance(values.begin(), sorted_end)
+
+
+def sort_ints(vector[int] values):
+    """Test sort using the default operator<.
+
+    >>> sort_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3])
+    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    """
+    sort(values.begin(), values.end())
+    return values
+
+
+def sort_ints_reverse(vector[int] values):
+    """Test sort using a standard library comparison function object.
+
+    >>> sort_ints_reverse([5, 7, 4, 2, 8, 6, 1, 9, 0, 3])
+    [9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
+    """
+    sort(values.begin(), values.end(), greater[int]())
+    return values
+
+
+def partial_sort_ints(vector[int] values, int k):
+    """
+    Test partial_sort using the default operator<.
+
+    >>> partial_sort_ints([4, 2, 3, 1, 5], 2)[:2]
+    [1, 2]
+    """
+    partial_sort(values.begin(), values.begin() + k, values.end())
+    return values
+
+
+def partial_sort_ints_reverse(vector[int] values, int k):
+    """
+    Test partial_sort using a standard library comparison function object.
+
+    >>> partial_sort_ints_reverse([4, 2, 3, 1, 5], 2)[:2]
+    [5, 4]
+    """
+    partial_sort(values.begin(), values.begin() + k, values.end(), greater[int]())
+    return values
+
+
+def partial_sort_ints2(vector[int] values, int k):
+    """
+    Test partial_sort_copy using the default operator<.
+
+    >>> partial_sort_ints2([4, 2, 3, 1, 5], 2)
+    [1, 2]
+    """
+    output = vector[int](2)
+    partial_sort_copy(values.begin(), values.end(), output.begin(), output.end())
+    return output
+
+
+def partial_sort_ints_reverse2(vector[int] values, int k):
+    """
+    Test partial_sort_copy using a standard library comparison function object.
+
+    >>> partial_sort_ints_reverse2([4, 2, 3, 1, 5], 2)
+    [5, 4]
+    """
+    output = vector[int](2)
+    partial_sort_copy(values.begin(), values.end(), output.begin(), output.end(), greater[int]())
+    return output
+
+
+cdef extern from *:
+    """
+    struct Employee
+    {
+        Employee() = default;
+        Employee(int age, std::string name): age(age), name(name) {}
+        int age;
+        std::string name;  // Does not participate in comparisons
+    };
+
+    bool operator<(const Employee& lhs, const Employee& rhs)
+    {
+        return lhs.age < rhs.age;
+    }
+    """
+    cppclass Employee:
+        Employee()
+        Employee(int, string)
+        int age
+        string name
+
+cdef bool Employee_greater(const Employee& lhs, const Employee& rhs):
+    return lhs.age > rhs.age
+
+def test_stable_sort():
+    """
+    Test stable_sort using cppreference example.
+
+    >>> test_stable_sort()
+    32, Arthur
+    108, Zaphod
+    108, Ford
+    108, Zaphod
+    108, Ford
+    32, Arthur
+    """
+    cdef vector[Employee] employees
+    employees.push_back(Employee(108, b"Zaphod"))
+    employees.push_back(Employee(32, b"Arthur"))
+    employees.push_back(Employee(108, b"Ford"))
+
+    stable_sort(employees.begin(), employees.end())
+
+    for e in employees:
+        print("%s, %s" % (e.age, (e.name).decode("ascii")))
+
+    stable_sort(employees.begin(), employees.end(), &Employee_greater)
+
+    for e in employees:
+        print("%s, %s" % (e.age, (e.name).decode("ascii")))
+
+
+def second_smallest(vector[int] values):
+    """
+    Test nth_element using the default operator<.
+
+    >>> second_smallest([5, 6, 4, 3, 2, 6, 7, 9, 3])
+    3
+    """
+    nth_element(values.begin(), values.begin() + 1, values.end())
+    return values[1]
+
+
+def second_largest(vector[int] values):
+    """
+    Test nth_element using a standard library comparison function object.
+
+    >>> second_largest([5, 6, 4, 3, 2, 6, 7, 9, 3])
+    7
+    """
+    nth_element(values.begin(), values.begin() + 1, values.end(), greater[int]())
+    return values[1]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_atomic.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_atomic.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_atomic.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_atomic.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,85 @@
+# mode: run
+# tag: cpp, cpp11, werror, no-cpp-locals
+
+from cython.operator cimport preincrement as incr, dereference as deref
+from libc.stdint cimport *
+
+from libcpp.atomic cimport atomic
+
+def int_test(int x):
+    """
+    >>> int_test(55)
+    3
+    >>> int_test(42)
+    3
+    >>> int_test(100000)
+    3
+    """
+    atom = new atomic[int](x)
+    try:
+        atom.store(0)
+        incr(deref(atom))
+        incr(deref(atom))
+        incr(deref(atom))
+        return atom.load()
+    finally:
+        del atom
+
+ctypedef atomic[int32_t] atomint32_t
+
+def typedef_test(int x):
+    """
+    >>> typedef_test(55)
+    3
+    >>> typedef_test(42)
+    3
+    >>> typedef_test(100000)
+    3
+    """
+    atom = new atomint32_t(x)
+    try:
+        atom.store(0)
+        incr(deref(atom))
+        incr(deref(atom))
+        incr(deref(atom))
+        return atom.load()
+    finally:
+        del atom
+
+def stack_allocation_test(int x):
+    """
+    >>> stack_allocation_test(55)
+    3
+    >>> stack_allocation_test(42)
+    3
+    >>> stack_allocation_test(100000)
+    3
+    """
+    cdef atomint32_t atom
+    atom.store(x)
+    try:
+        atom.store(0)
+        incr(atom)
+        incr(atom)
+        incr(atom)
+        return atom.load()
+    finally:
+        pass
+
+def nogil_int_test(int x):
+    """
+    >>> nogil_int_test(55)
+    55
+    >>> nogil_int_test(42)
+    42
+    >>> nogil_int_test(100000)
+    100000
+    """
+    with nogil:
+        atom = new atomic[int](0)
+    try:
+        with nogil:
+            atom.store(x)
+        return atom.load()
+    finally:
+        del atom
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_conversion.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_conversion.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_conversion.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_conversion.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,8 +1,9 @@
 # mode: run
-# tag: cpp, werror
-# distutils: extra_compile_args=-std=c++0x
+# tag: cpp, werror, cpp11
 
 import sys
+from collections import defaultdict
+
 from libcpp.map cimport map
 from libcpp.unordered_map cimport unordered_map
 from libcpp.set cimport set as cpp_set
@@ -16,7 +17,7 @@
 py_xrange = xrange
 py_unicode = unicode
 
-cdef string add_strings(string a, string b):
+cdef string add_strings(string a, string b) except *:
     return a + b
 
 def normalize(bytes b):
@@ -72,6 +73,16 @@
     """
     return add_strings(a, b)
 
+def test_c_string_convert(char *c_string):
+    """
+    >>> normalize(test_c_string_convert("abc".encode('ascii')))
+    'abc'
+    """
+    cdef string s
+    with nogil:
+        s = c_string
+    return s
+
 def test_int_vector(o):
     """
     >>> test_int_vector([1, 2, 3])
@@ -88,6 +99,23 @@
     cdef vector[int] v = o
     return v
 
+cdef vector[int] takes_vector(vector[int] x):
+    return x
+
+def test_list_literal_to_vector():
+    """
+    >>> test_list_literal_to_vector()
+    [1, 2, 3]
+    """
+    return takes_vector([1, 2, 3])
+
+def test_tuple_literal_to_vector():
+    """
+    >>> test_tuple_literal_to_vector()
+    [1, 2, 3]
+    """
+    return takes_vector((1, 2, 3))
+
 def test_string_vector(s):
     """
     >>> list(map(normalize, test_string_vector('ab cd ef gh'.encode('ascii'))))
@@ -134,10 +162,12 @@
     Traceback (most recent call last):
     ...
     OverflowError: ...
+
+    "TypeError: an integer is required" on CPython
     >>> test_typedef_vector([1, 2, None])       #doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef vector[my_int] v = o
     return v
@@ -184,22 +214,36 @@
 
 def test_map(o):
     """
-    >>> test_map({1: 1.0, 2: 0.5, 3: 0.25})
+    >>> d = {1: 1.0, 2: 0.5, 3: 0.25}
+    >>> test_map(d)
+    {1: 1.0, 2: 0.5, 3: 0.25}
+    >>> dd = defaultdict(float)
+    >>> dd.update(d)
+    >>> test_map(dd)  # try with a non-dict
     {1: 1.0, 2: 0.5, 3: 0.25}
     """
     cdef map[int, double] m = o
     return m
 
 def test_unordered_map(o):
-   """
-   >>> d = test_map({1: 1.0, 2: 0.5, 3: 0.25})
-   >>> sorted(d)
-   [1, 2, 3]
-   >>> (d[1], d[2], d[3])
-   (1.0, 0.5, 0.25)
-   """
-   cdef unordered_map[int, double] m = o
-   return m
+    """
+    >>> d = {1: 1.0, 2: 0.5, 3: 0.25}
+    >>> m = test_map(d)
+    >>> sorted(m)
+    [1, 2, 3]
+    >>> (m[1], m[2], m[3])
+    (1.0, 0.5, 0.25)
+
+    >>> dd = defaultdict(float)
+    >>> dd.update(d)
+    >>> m = test_map(dd)
+    >>> sorted(m)
+    [1, 2, 3]
+    >>> (m[1], m[2], m[3])
+    (1.0, 0.5, 0.25)
+    """
+    cdef unordered_map[int, double] m = o
+    return m
 
 def test_nested(o):
     """
@@ -230,3 +274,14 @@
     """
     cdef map[Color, Color] m = o
     return m
+
+cdef map[unsigned int, unsigned int] takes_map(map[unsigned int, unsigned int] m):
+    return m
+
+def test_dict_literal_to_map():
+    """
+    >>> test_dict_literal_to_map()
+    {1: 1}
+    """
+    return takes_map({1: 1})  # https://github.com/cython/cython/pull/4228
+                              # DictNode could not be converted directly
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_cpp11.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_cpp11.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_cpp11.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_cpp11.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,203 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+import sys
+from libcpp.unordered_map cimport unordered_map
+from libcpp.unordered_set cimport unordered_set
+from libcpp.vector cimport vector
+from libcpp.queue cimport queue
+from libcpp.queue cimport priority_queue
+from libcpp.vector cimport vector
+from libcpp.pair cimport pair
+from libcpp.map cimport map
+from libcpp.set cimport set
+from libcpp.deque cimport deque
+
+
+def test_vector_functionality():
+    """
+    >>> test_vector_functionality()
+    'pass'
+    """
+    cdef:
+        vector[int] int_vector = vector[int]()
+        int* data
+        const int* const_data
+    int_vector.push_back(77)
+    data = int_vector.data()
+    const_data = int_vector.const_data()
+    assert data[0] == 77
+    assert const_data[0] == 77
+    return "pass"
+
+
+def test_queue_functionality():
+    """
+    >>> test_queue_functionality()
+    'pass'
+    """
+    cdef:
+        queue[int] int_queue = queue[int]()
+        queue[int] int_queue2 = queue[int]()
+    int_queue.push(77)
+    int_queue.swap(int_queue2)
+    assert int_queue.size() == 0
+    assert int_queue2.size() == 1
+    return "pass"
+
+
+def test_deque_functionality():
+    """
+    >>> test_deque_functionality()
+    'pass'
+    """
+    cdef:
+        deque[int] int_deque = deque[int]()
+    int_deque.push_back(77)
+    int_deque.shrink_to_fit()
+    return "pass"
+
+
+def test_priority_queue_functionality():
+    """
+    >>> test_priority_queue_functionality()
+    'pass'
+    """
+    cdef:
+        priority_queue[int] int_queue = priority_queue[int]()
+        priority_queue[int] int_queue2 = priority_queue[int]()
+    int_queue.push(77)
+    int_queue.swap(int_queue2)
+    assert int_queue.size() == 0
+    assert int_queue2.size() == 1
+    return "pass"
+
+
+def test_set_functionality():
+    """
+    >>> test_set_functionality()
+    'pass'
+    """
+    cdef:
+        set[int] int_set
+        set[int] int_set2
+    int_set2.insert(77)
+    int_set2.insert(66)
+    int_set.insert(int_set2.const_begin(), int_set2.const_end())
+    assert int_set.size() == 2
+    assert int_set.erase(int_set.const_begin(), int_set.const_end()) == int_set.end()
+    return "pass"
+
+
+def test_map_functionality():
+    """
+    >>> test_map_functionality()
+    'pass'
+    """
+    cdef:
+        map[int, const void*] int_map
+        const void* data
+    int_map[77] = NULL
+    data = int_map.const_at(77)
+    return "pass"
+
+
+def test_unordered_set_functionality():
+    """
+    >>> test_unordered_set_functionality()
+    'pass'
+    """
+    cdef:
+        unordered_set[int] int_set = unordered_set[int]()
+        unordered_set[int] int_set2
+        unordered_set[int].iterator iterator = int_set.begin()
+    int_set.insert(1)
+    assert int_set.size() == 1
+    int_set.erase(unordered_set[int].const_iterator(int_set.begin()), unordered_set[int].const_iterator(int_set.end()))
+    assert int_set.size() == 0
+    int_set.insert(1)
+    assert int_set.erase(1) == 1 # returns number of elements erased
+    assert int_set.size() == 0
+    int_set.insert(1)
+    iterator = int_set.find(1)
+    assert int_set.erase(iterator) == int_set.end()
+
+    int_set2.insert(3)
+    int_set2.insert(5)
+    int_set.insert(int_set2.begin(), int_set2.end())
+    assert int_set.size() == 2
+
+    if sys.platform != 'darwin':
+        int_set.max_load_factor(0.5)
+        assert int_set.max_load_factor() == 0.5
+    int_set.rehash(20)
+    int_set.reserve(20)
+
+    int_set.bucket_size(0)
+    int_set.bucket_count()
+    int_set.max_bucket_count()
+    int_set.bucket(3)
+    assert int_set.load_factor() > 0
+    return "pass"
+
+
+cdef extern from "cpp_unordered_map_helper.h":
+    cdef cppclass IntVectorHash:
+        pass
+
+
+def test_unordered_map_functionality():
+    """
+    >>> test_unordered_map_functionality()
+    'pass'
+    """
+    cdef:
+        unordered_map[int, int] int_map = unordered_map[int,int]()
+        pair[int, int] pair_insert = pair[int, int](1, 2)
+        unordered_map[int,int].iterator iterator = int_map.begin()
+        pair[unordered_map[int,int].iterator, bint] pair_iter  = int_map.insert(pair_insert)
+        unordered_map[int, int] int_map2
+        unordered_map[int, int*] intptr_map
+        const int* intptr
+        unordered_map[vector[int], int, IntVectorHash] int_vector_map
+        vector[int] intvec
+    assert int_map[1] == 2
+    assert int_map.size() == 1
+    assert int_map.erase(1) == 1 # returns number of elements erased
+    assert int_map.size() == 0
+    int_map[1] = 2
+    assert int_map.size() == 1
+    assert int_map[1] == 2
+    iterator = int_map.find(1)
+    assert int_map.erase(iterator) == int_map.end()
+
+    int_map2[1] = 2
+    int_map2[3] = 3
+    int_map.clear()
+    int_map.insert(int_map2.begin(), int_map2.end())
+    assert int_map.size() == 2
+    assert int_map.erase(unordered_map[int,int].const_iterator(int_map.begin()), unordered_map[int,int].const_iterator(int_map.end())) == int_map.end()
+
+    int_map.max_load_factor(0.5)
+    assert int_map.max_load_factor() == 0.5
+    int_map.rehash(20)
+    int_map.reserve(20)
+
+    int_map[3] = 3
+    int_map.bucket_size(0)
+    int_map.bucket_count()
+    int_map.max_bucket_count()
+    int_map.bucket(3)
+    assert int_map.load_factor() > 0
+
+    intptr_map[0] = NULL
+    intptr = intptr_map.const_at(0)
+
+    intvec = [1, 2]
+    int_vector_map[intvec] = 3
+    intvec = [4, 5]
+    int_vector_map[intvec] = 6
+    assert int_vector_map[intvec] == 6
+    intvec = [1, 2]
+    assert int_vector_map[intvec] == 3
+    return "pass"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_forward_list.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_forward_list.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_forward_list.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_forward_list.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,79 @@
+# mode: run
+# tag: cpp, werror, cpp11, no-cpp-locals
+
+from cython.operator cimport dereference as deref
+from cython.operator cimport preincrement as incr
+
+from libcpp.forward_list cimport forward_list
+from libcpp cimport bool as cbool
+
+
+def simple_iteration_test(L):
+    """
+    >>> iteration_test([1,2,4,8])
+    8
+    4
+    2
+    1
+    >>> iteration_test([8,4,2,1])
+    1
+    2
+    4
+    8
+    """
+    cdef forward_list[int] l
+    for a in L:
+        l.push_front(a)
+    for a in l:
+        print(a)
+
+def iteration_test(L):
+    """
+    >>> iteration_test([1,2,4,8])
+    8
+    4
+    2
+    1
+    >>> iteration_test([8,4,2,1])
+    1
+    2
+    4
+    8
+    """
+    l = new forward_list[int]()
+    try:
+        for a in L:
+            l.push_front(a)
+        it = l.begin()
+        while it != l.end():
+            a = deref(it)
+            incr(it)
+            print(a)
+    finally:
+        del l
+
+def test_value_type(x):
+    """
+    >>> test_value_type(2)
+    2.0
+    >>> test_value_type(2.5)
+    2.5
+    """
+    cdef forward_list[double].value_type val = x
+    return val
+
+def test_value_type_complex(x):
+    """
+    >>> test_value_type_complex(2)
+    (2+0j)
+    """
+    cdef forward_list[double complex].value_type val = x
+    return val
+
+
+#  Tests GitHub issue #1788.
+cdef cppclass MyForwardList[T](forward_list):
+    pass
+
+cdef cppclass Ints(MyForwardList[int]):
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_function.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_function.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_function.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_function.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,86 @@
+# mode: run
+# tag: cpp, cpp11
+
+from libcpp.functional cimport function
+cimport cpp_function_lib
+
+def test_simple_function():
+    '''
+    >>> test_simple_function()
+    6.0
+    '''
+    return cpp_function_lib.add_one(2.0, 3)
+
+
+def test_AddAnotherFunctor(n):
+    '''
+    >>> test_AddAnotherFunctor(5.0)
+    10.0
+    '''
+    return cpp_function_lib.AddAnotherFunctor(5.0).call(2.0, 3)
+
+
+cdef class FunctionKeeper:
+    """
+    >>> fk = FunctionKeeper('add_one')
+    >>> fk(2.0, 3)
+    6.0
+    >>> fk = FunctionKeeper('add_two')
+    >>> fk(2.0, 3)
+    7.0
+    >>> fk = FunctionKeeper('AddAnotherFunctor5')
+    >>> fk(2.0, 3)
+    10.0
+    >>> fk = FunctionKeeper('default')
+    >>> bool(fk)
+    False
+    >>> fk(2.0, 3)
+    Traceback (most recent call last):
+    ...
+    RuntimeError: Trying to call undefined function!
+    >>> fk.set_function('AddAnotherFunctor5')
+    >>> fk(2.0, 3)
+    10.0
+    >>> bool(fk)
+    True
+    >>> fk.set_function('NULL')
+    >>> bool(fk)
+    False
+    """
+    cdef cpp_function_lib.FunctionKeeper* function_keeper
+    
+    cdef function[double(double, int)]* _get_function_ptr_from_name(self, function_name):
+        cdef function[double(double, int)] *f
+        
+        if function_name == 'add_one':
+            f = new function[double(double, int)](cpp_function_lib.add_one)
+        elif function_name == 'add_two':
+            f = new function[double(double, int)](cpp_function_lib.add_two)
+        elif function_name == 'AddAnotherFunctor5':
+            f = new function[double(double, int)]()
+            f[0] = cpp_function_lib.AddAnotherFunctor(5.0)
+        elif function_name == 'NULL':
+            f = new function[double(double, int)](NULL)
+        elif function_name == 'default':
+            f = new function[double(double, int)]()
+            
+        return f
+   
+    def __cinit__(self, function_name):
+        cdef function[double(double, int)] *f = self._get_function_ptr_from_name(function_name)
+        self.function_keeper = new cpp_function_lib.FunctionKeeper(f[0])
+        del f
+
+    def __dealloc__(self):
+        del self.function_keeper
+
+    def __call__(self, a, b):
+        return self.function_keeper.call_function(a, b)
+
+    def __bool__(self):
+        return  self.function_keeper.get_function()
+
+    def set_function(self, function_name):
+        cdef function[double(double, int)] *f = self._get_function_ptr_from_name(function_name)
+        self.function_keeper.set_function(f[0])
+        del f
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_list.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_list.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_list.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_list.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,158 @@
+# mode: run
+# tag: cpp, werror, no-cpp-locals
+
+from cython.operator cimport dereference as deref
+from cython.operator cimport preincrement as incr
+
+from libcpp.list cimport list as cpp_list
+from libcpp cimport bool as cbool
+
+
+def simple_test(double x):
+    """
+    >>> simple_test(55)
+    3
+    """
+    l = new cpp_list[double]()
+    try:
+        l.push_back(1.0)
+        l.push_back(x)
+        from math import pi
+        l.push_back(pi)
+        return l.size()
+    finally:
+        del l
+
+def pylist_test(L):
+    """
+    >>> pylist_test([1,2,4,8])
+    (4, 4)
+    >>> pylist_test([])
+    (0, 0)
+    >>> pylist_test([-1] * 1000)
+    (1000, 1000)
+    """
+    l = new cpp_list[int]()
+    try:
+        for a in L:
+            l.push_back(a)
+        return len(L), l.size()
+    finally:
+        del l
+
+
+def iteration_test(L):
+    """
+    >>> iteration_test([1,2,4,8])
+    1
+    2
+    4
+    8
+    """
+    l = new cpp_list[int]()
+    try:
+        for a in L:
+            l.push_back(a)
+        it = l.begin()
+        while it != l.end():
+            a = deref(it)
+            incr(it)
+            print(a)
+    finally:
+        del l
+
+def reverse_iteration_test(L):
+    """
+    >>> reverse_iteration_test([1,2,4,8])
+    8
+    4
+    2
+    1
+    """
+    l = new cpp_list[int]()
+    try:
+        for a in L:
+            l.push_back(a)
+        it = l.rbegin()
+        while it != l.rend():
+            a = deref(it)
+            incr(it)
+            print(a)
+    finally:
+        del l
+
+def nogil_test(L):
+    """
+    >>> nogil_test([1,2,3])
+    3
+    """
+    cdef int a
+    with nogil:
+        l = new cpp_list[int]()
+    try:
+        for a in L:
+            with nogil:
+                l.push_back(a)
+        return l.size()
+    finally:
+        del l
+
+
+cdef list to_pylist(cpp_list[int]& l):
+    cdef list L = []
+    it = l.begin()
+    while it != l.end():
+        L.append(deref(it))
+        incr(it)
+    return L
+
+
+def item_ptr_test(L, int x):
+    """
+    >>> item_ptr_test(range(10), 100)
+    [100, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    """
+    cdef cpp_list[int] l = L
+    cdef int* li_ptr = &l.front()
+    li_ptr[0] = x
+    return to_pylist(l)
+
+def test_value_type(x):
+    """
+    >>> test_value_type(2)
+    2.0
+    >>> test_value_type(2.5)
+    2.5
+    """
+    cdef cpp_list[double].value_type val = x
+    return val
+
+def test_value_type_complex(x):
+    """
+    >>> test_value_type_complex(2)
+    (2+0j)
+    """
+    cdef cpp_list[double complex].value_type val = x
+    return val
+
+def test_insert():
+    """
+    >>> test_insert()
+    """
+    cdef cpp_list[int] l
+    cdef cpp_list[int].size_type count = 5
+    cdef int value = 0
+
+    l.insert(l.end(), count, value)
+
+    assert l.size() == count
+    for element in l:
+        assert element == value, '%s != %s' % (element, count)
+
+
+#  Tests GitHub issue #1788.
+cdef cppclass MyList[T](cpp_list):
+    pass
+
+cdef cppclass Ints(MyList[int]):
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_map.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_map.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_map.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_map.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,137 @@
+# mode: run
+# tag: cpp, cpp11
+
+# cython: language_level=3
+
+from libcpp.map cimport map
+from libcpp.unordered_map cimport unordered_map
+from libcpp.utility cimport pair
+
+def test_map_insert(vals):
+    """
+    >>> test_map_insert([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef map[int,int] m = map[int, int]()
+    cdef pair[map[int, int].iterator, bint] ret
+    for v in vals:
+        ret = m.insert(v)
+    return [ (item.first, item.second) for item in m ]
+
+def test_map_insert_it(vals):
+    """
+    >>> test_map_insert_it([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    cdef map[int,int] m = map[int,int]()
+    for k, v in vals:
+        um.insert(pair[int,int](k, v))
+    m.insert(um.begin(), um.end())
+    return [ (item.first, item.second) for item in m ]
+
+def test_map_count(vals, to_find):
+    """
+    >>> test_map_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    1
+    >>> test_map_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    1
+    """
+    cdef map[int,int] m = map[int,int]()
+    for v in vals:
+        m.insert(v)
+    return m.count(to_find)
+
+def test_map_erase(vals, int to_remove):
+    """
+    >>> test_map_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (3, 3)]
+    >>> test_map_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef map[int,int] m = map[int,int]()
+    cdef size_t ret
+    for v in vals:
+        m.insert(v)
+    ret = m.erase(to_remove)
+    return [ (item.first, item.second) for item in m ]
+
+def test_map_find_erase(vals, to_remove):
+    """
+    >>> test_map_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (3, 3)]
+    >>> test_map_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef map[int,int] m = map[int,int]()
+    cdef map[int,int].iterator it
+    for v in vals:
+        m.insert(v)
+    it = m.find(to_remove)
+    it = m.erase(it)
+    return [ (item.first, item.second) for item in m ]
+
+
+def test_unordered_map_insert(vals):
+    """
+    >>> test_unordered_map_insert([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    cdef pair[unordered_map[int,int].iterator, bint] ret
+    for v in vals:
+        ret = um.insert(v)
+    return sorted([ (item.first, item.second) for item in um ])
+
+def test_unordered_map_insert_it(vals):
+    """
+    >>> test_unordered_map_insert_it([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef map[int,int] m = map[int,int]()
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    for v in vals:
+        m.insert(v)
+    um.insert(m.begin(), m.end())
+    return sorted([ (item.first, item.second) for item in um ])
+
+def test_unordered_map_count(vals, to_find):
+    """
+    >>> test_unordered_map_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    1
+    >>> test_unordered_map_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    1
+    """
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    for v in vals:
+        um.insert(v)
+    return um.count(to_find)
+
+def test_unordered_map_erase(vals, int to_remove):
+    """
+    >>> test_unordered_map_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (3, 3)]
+    >>> test_unordered_map_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    cdef size_t ret
+    for v in vals:
+        um.insert(v)
+    ret = um.erase(to_remove)
+    return sorted([ (item.first, item.second) for item in um ])
+
+def test_unordered_map_find_erase(vals, to_remove):
+    """
+    >>> test_unordered_map_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (3, 3)]
+    >>> test_unordered_map_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef unordered_map[int,int] um = unordered_map[int,int]()
+    cdef unordered_map[int,int].iterator it
+    for v in vals:
+        um.insert(v)
+    it = um.find(to_remove)
+    it = um.erase(it)
+    return sorted([ item for item in um ])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_multimap.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_multimap.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_multimap.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_multimap.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,135 @@
+# mode: run
+# tag: cpp, cpp11
+
+# cython: language_level=3
+
+from libcpp.map cimport multimap
+from libcpp.unordered_map cimport unordered_multimap
+from libcpp.utility cimport pair
+
+def test_multimap_insert(vals):
+    """
+    >>> test_multimap_insert([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (2, 2), (3, 3)]
+    """
+    cdef multimap[int,int] mm = multimap[int, int]()
+    cdef multimap[int, int].iterator it
+    for v in vals:
+        it = mm.insert(v)
+    return [ (item.first, item.second) for item in mm ]
+
+def test_multimap_insert_it(vals):
+    """
+    >>> test_multimap_insert_it([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (2, 2), (3, 3)]
+    """
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    cdef multimap[int,int] mm = multimap[int,int]()
+    for k, v in vals:
+        umm.insert(pair[int,int](k, v))
+    mm.insert(umm.begin(), umm.end())
+    return [ (item.first, item.second) for item in mm ]
+
+def test_multimap_count(vals, to_find):
+    """
+    >>> test_multimap_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    1
+    >>> test_multimap_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    2
+    """
+    cdef multimap[int,int] mm = multimap[int,int]()
+    for v in vals:
+        mm.insert(v)
+    return mm.count(to_find)
+
+def test_multimap_erase(vals, int to_remove):
+    """
+    >>> test_multimap_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (2, 2), (3, 3)]
+    >>> test_multimap_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef multimap[int,int] mm = multimap[int,int]()
+    for v in vals:
+        mm.insert(v)
+    cdef size_t ret = mm.erase(to_remove)
+    return [ (item.first, item.second) for item in mm ]
+
+def test_multimap_find_erase(vals, to_remove):
+    """
+    >>> test_multimap_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (2, 2), (3, 3)]
+    >>> test_multimap_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef multimap[int,int] mm = multimap[int,int]()
+    cdef multimap[int,int].iterator it
+    for v in vals:
+        mm.insert(v)
+    it = mm.find(to_remove)
+    it = mm.erase(it)
+    return [ (item.first, item.second) for item in mm ]
+
+
+def test_unordered_multimap_insert(vals):
+    """
+    >>> test_unordered_multimap_insert([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (2, 2), (3, 3)]
+    """
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    cdef unordered_multimap[int,int].iterator it
+    for v in vals:
+        it = umm.insert(v)
+    return sorted([ (item.first, item.second) for item in umm ])
+
+def test_unordered_multimap_insert_it(vals):
+    """
+    >>> test_unordered_multimap_insert_it([(1,1),(2,2),(2,2),(3,3),(-1,-1)])
+    [(-1, -1), (1, 1), (2, 2), (2, 2), (3, 3)]
+    """
+    cdef multimap[int,int] mm = multimap[int,int]()
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    for v in vals:
+        mm.insert(v)
+    umm.insert(mm.begin(), mm.end())
+    return sorted([ (item.first, item.second) for item in umm ])
+
+def test_unordered_multimap_count(vals, to_find):
+    """
+    >>> test_unordered_multimap_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    1
+    >>> test_unordered_multimap_count([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    2
+    """
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    for v in vals:
+        umm.insert(v)
+    return umm.count(to_find)
+
+def test_unordered_multimap_erase(vals, int to_remove):
+    """
+    >>> test_unordered_multimap_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (2, 2), (3, 3)]
+    >>> test_unordered_multimap_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (3, 3)]
+    """
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    for v in vals:
+        umm.insert(v)
+    cdef size_t ret = umm.erase(to_remove)
+    return sorted([ (item.first, item.second) for item in umm ])
+
+def test_unordered_multimap_find_erase(vals, to_remove):
+    """
+    >>> test_unordered_multimap_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 1)
+    [(-1, -1), (2, 2), (2, 2), (3, 3)]
+    >>> test_unordered_multimap_find_erase([(1,1),(2,2),(2,2),(3,3),(-1,-1)], 2)
+    [(-1, -1), (1, 1), (2, 2), (3, 3)]
+    """
+    cdef unordered_multimap[int,int] umm = unordered_multimap[int,int]()
+    cdef unordered_multimap[int,int].iterator it
+    for v in vals:
+        umm.insert(v)
+    it = umm.find(to_remove)
+    it = umm.erase(it)
+    return sorted([ item for item in umm ])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_multiset.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_multiset.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_multiset.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_multiset.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,145 @@
+# mode: run
+# tag: cpp, cpp11
+
+# cython: language_level=3
+
+from libcpp.set cimport multiset
+from libcpp.unordered_set cimport unordered_multiset
+
+def test_multiset_insert(vals):
+    """
+    >>> test_multiset_insert([1,2,2,3, -1])
+    [-1, 1, 2, 2, 3]
+    """
+    cdef multiset[int] ms = multiset[int]()
+    cdef multiset[int].iterator it
+    for v in vals:
+        it = ms.insert(v)
+    return [ item for item in ms ]
+
+def test_multiset_insert_it(vals):
+    """
+    >>> test_multiset_insert_it([1,2,2,3, -1])
+    [-1, 1, 2, 2, 3]
+    """
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    cdef multiset[int] ms = multiset[int]()
+    for v in vals:
+        ums.insert(v)
+    ms.insert(ums.begin(), ums.end())
+    return [ item for item in ms ]
+
+def test_multiset_count(vals, to_find):
+    """
+    >>> test_multiset_count([1,2,2,3, -1], 1)
+    1
+    >>> test_multiset_count([1,2,2,3, -1], 2)
+    2
+    """
+    cdef multiset[int] ms = multiset[int]()
+    for v in vals:
+        ms.insert(v)
+    return ms.count(to_find)
+
+def test_multiset_erase(vals, int to_remove):
+    """
+    >>> test_multiset_erase([1,2,2,3, -1], 1)
+    [-1, 2, 2, 3]
+    >>> test_multiset_erase([1,2,2,3, -1], 2)  # removes both copies of 2
+    [-1, 1, 3]
+    """
+    cdef multiset[int] ms = multiset[int]()
+    cdef size_t ret
+    for v in vals:
+        ms.insert(v)
+    ret = ms.erase(to_remove)
+    return [ item for item in ms ]
+
+def test_multiset_find_erase(vals, to_remove):
+    """
+    >>> test_multiset_find_erase([1,2,2,3, -1], 1)
+    [-1, 2, 2, 3]
+    >>> test_multiset_find_erase([1,2,2,3, -1], 2)  # removes a single copy of 2
+    [-1, 1, 2, 3]
+    """
+    cdef multiset[int] ms = multiset[int]()
+    cdef multiset[int].iterator it
+    for v in vals:
+        ms.insert(v)
+    it = ms.find(to_remove)
+    it = ms.erase(it)
+    return [ item for item in ms ]
+
+
+def test_unordered_multiset_insert(vals):
+    """
+    >>> test_unordered_multiset_insert([1,2,2,3, -1])
+    [-1, 1, 2, 2, 3]
+    """
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    cdef unordered_multiset[int].iterator it
+    for v in vals:
+        it = ums.insert(v)
+    return sorted([ item for item in ums ])
+
+def test_unordered_multiset_insert_it(vals):
+    """
+    >>> test_unordered_multiset_insert_it([1,2,2,3, -1])
+    [-1, 1, 2, 2, 3]
+    """
+    cdef multiset[int] ms = multiset[int]()
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    for v in vals:
+        ms.insert(v)
+    ums.insert(ms.begin(), ms.end())
+    return sorted([ item for item in ums ])
+
+def test_unordered_multiset_count(vals, to_find):
+    """
+    >>> test_unordered_multiset_count([1,2,2,3, -1], 1)
+    1
+    >>> test_unordered_multiset_count([1,2,2,3, -1], 2)
+    2
+    """
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    for v in vals:
+        ums.insert(v)
+    return ums.count(to_find)
+
+def test_unordered_multiset_erase(vals, int to_remove):
+    """
+    >>> test_unordered_multiset_erase([1,2,2,3, -1], 1)
+    [-1, 2, 2, 3]
+    >>> test_unordered_multiset_erase([1,2,2,3, -1], 2)  # removes both copies of 2
+    [-1, 1, 3]
+    """
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    cdef size_t ret
+    for v in vals:
+        ums.insert(v)
+    ret = ums.erase(to_remove)
+    return sorted([ item for item in ums ])
+
+def test_unordered_multiset_find_erase(vals, to_remove):
+    """
+    >>> test_unordered_multiset_find_erase([1,2,2,3, -1], 1)
+    [-1, 2, 2, 3]
+    >>> test_unordered_multiset_find_erase([1,2,2,3, -1], 2)  # removes a single copy of 2
+    [-1, 1, 2, 3]
+    """
+    cdef unordered_multiset[int] ums = unordered_multiset[int]()
+    cdef unordered_multiset[int].iterator it
+    for v in vals:
+        ums.insert(v)
+    it = ums.find(to_remove)
+    it = ums.erase(it)
+    return sorted([ item for item in ums ])
+
+
+def test_unordered_multiset_misc():
+    """
+    >>> test_unordered_multiset_misc()
+    """
+    cdef unordered_multiset[int] ms = unordered_multiset[int]()
+    ms.insert(1)
+    assert ms.load_factor() > 0
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_numeric_ops_cpp17.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_numeric_ops_cpp17.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_numeric_ops_cpp17.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_numeric_ops_cpp17.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,277 @@
+# mode: run
+# tag: cpp, werror, cpp17, cppexecpolicies
+
+from libcpp.numeric cimport (reduce, transform_reduce, inclusive_scan, 
+                             exclusive_scan, transform_inclusive_scan, 
+                             transform_exclusive_scan)
+from libcpp.execution cimport seq
+from libcpp.vector cimport vector
+
+# Subtracts two integers.
+cdef int subtract_integers(int lhs, int rhs):
+    return lhs - rhs
+
+# Adds two integers.
+cdef int add_integers(int lhs, int rhs):
+    return lhs + rhs
+
+# Multiplies two integers.
+cdef int multiply_integers(int lhs, int rhs):
+    return lhs * rhs
+
+# Multiplies a integer with 2
+cdef int multiply_with_2(int val):
+    return 2*val
+
+def test_reduce(vector[int] v, int init):
+    """
+    Test reduce.
+     0 + 1 = 1
+     1 + 2 = 3
+     3 + 3 = 6
+    >>> test_reduce([1, 2, 3], 0)
+    6
+    """
+    return reduce(v.begin(), v.end(), init)
+
+def test_reduce_with_bin_op(vector[int] v, int init):
+    """
+    Test reduce with a binary operation (subtraction). 
+     0 - 1 = -1
+    -1 - 2 = -3
+    -3 - 3 = -6
+    >>> test_reduce_with_bin_op([1, 2, 3], 0)
+    -6
+    """
+    return reduce(v.begin(), v.end(), init, subtract_integers)
+
+# def test_reduce_with_execpolicy(vector[int] v, int init):
+#     """
+#     Test reduce with execution policy. 
+#      0 + 1 = 1
+#      1 + 2 = 3
+#      3 + 3 = 6
+#     >>> test_reduce_with_execpolicy([1, 2, 3], 0)
+#     6
+#     """
+#     return reduce(seq, v.begin(), v.end(), init)
+
+def test_reduce_with_bin_op_and_execpolicy(vector[int] v, int init):
+    """
+    Test reduce with execution policy and a binary operation (subtraction). 
+     0 - 1 = -1
+    -1 - 2 = -3
+    -3 - 3 = -6
+    >>> test_reduce_with_bin_op_and_execpolicy([1, 2, 3], 0)
+    -6
+    """
+    return reduce(seq, v.begin(), v.end(), init, subtract_integers)
+
+def test_transform_reduce(vector[int] v1, vector[int] v2, int init):
+    """
+    Test transform_reduce
+    >>> test_transform_reduce([1, 2, 3], [1, 2, 3], 0)
+    14
+    """
+    return transform_reduce(v1.begin(), v1.end(), v2.begin(), init)
+
+def test_transform_reduce_with_bin_red_op_and_bin_tran_op(vector[int] v1, vector[int] v2, int init):
+    """
+    Test transform_reduce with a binary reduce and transform operations
+    >>> test_transform_reduce_with_bin_red_op_and_bin_tran_op([1, 2, 3], [1, 2, 3], 0)
+    14
+    """
+    return transform_reduce(v1.begin(), v1.end(), v2.begin(), init, add_integers, multiply_integers)
+
+def test_transform_reduce_with_bin_op_and_unary_op(vector[int] v1, vector[int] v2, int init):
+    """
+    Test transform_reduce with a binary reduction and a unary transform operation
+    >>> test_transform_reduce_with_bin_op_and_unary_op([1, 2, 3], [1, 2, 3], 0)
+    12
+    """
+    return transform_reduce(v1.begin(), v1.end(), init, add_integers, multiply_with_2)
+
+# def test_transform_reduce_with_execpolicy(vector[int] v1, vector[int] v2, int init):
+#     """
+#     Test transform_reduce with a execution policy
+#     >>> test_transform_reduce_with_execpolicy([1, 2, 3], [1, 2, 3], 0)
+#     14
+#     """
+#     return transform_reduce(seq, v1.begin(), v1.end(), v2.begin(), init)
+
+def test_transform_reduce_with_execpolicy_bin_red_op_and_bin_tran_op(vector[int] v1, vector[int] v2, int init):
+    """
+    Test transform_reduce with a execution policy and binary reduce and transform operations
+    >>> test_transform_reduce_with_execpolicy_bin_red_op_and_bin_tran_op([1, 2, 3], [1, 2, 3], 0)
+    14
+    """
+    return transform_reduce(seq, v1.begin(), v1.end(), v2.begin(), init, add_integers, multiply_integers)
+
+# def test_transform_reduce_with_execpolicy_bin_op_and_unary_op(vector[int] v1, vector[int] v2, int init):
+#     """
+#     Test transform_reduce with a execution policy and binary reduction and a unary transform operation
+#     >>> test_transform_reduce_with_execpolicy_bin_op_and_unary_op([1, 2, 3], [1, 2, 3], 0)
+#     12
+#     """
+#     return transform_reduce(seq, v1.begin(), v1.end(), v2.begin(), init, add_integers, multiply_with_2)
+
+def test_inclusive_scan(vector[int] v):
+    """
+    Test inclusive_scan
+    >>> test_inclusive_scan([1, 2, 3, 4])
+    [1, 3, 6, 10]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    inclusive_scan(v.begin(), v.end(), out.begin())
+    return out
+
+# def test_inclusive_scan_with_execpolicy(vector[int] v):
+#     """
+#     Test inclusive_scan with a execution policy
+#     >>> test_inclusive_scan_with_execpolicy([1, 2, 3, 4])
+#     [1, 3, 6, 10]
+#     """
+#     cdef vector[int] out
+#     inclusive_scan(seq, v.begin(), v.end(), out.begin())
+#     return out
+
+def test_inclusive_scan_with_bin_op(vector[int] v):
+    """
+    Test inclusive_scan with a binary operation
+    >>> test_inclusive_scan_with_bin_op([1, 2, 3, 4])
+    [1, 3, 6, 10]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    inclusive_scan(v.begin(), v.end(), out.begin(), add_integers)
+    return out
+
+# def test_inclusive_scan_with_execpolicy_and_bin_op(vector[int] v):
+#     """
+#     Test inclusive_scan with a execution policy and a binary operation
+#     >>> test_inclusive_scan_with_execpolicy_and_bin_op([1, 2, 3, 4])
+#     [1, 3, 6, 10]
+#     """
+#     cdef vector[int] out
+#     inclusive_scan(seq, v.begin(), v.end(), out.begin(), add_integers)
+#     return out
+
+def test_inclusive_scan_with_bin_op_and_init(vector[int] v, int init):
+    """
+    Test inclusive_scan with a binary operation and a initial value
+    >>> test_inclusive_scan_with_bin_op_and_init([1, 2, 3, 4], 0)
+    [1, 3, 6, 10]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    inclusive_scan(v.begin(), v.end(), out.begin(), add_integers, init)
+    return out
+
+# def test_inclusive_scan_with_execpolicy_bin_op_and_init(vector[int] v, int init):
+#     """
+#     Test inclusive_scan with a execution policy, a binary operation and a initial value
+#     >>> test_inclusive_scan_with_execpolicy_bin_op_and_init([1, 2, 3, 4], 0)
+#     [1, 3, 6, 10]
+#     """
+#     cdef vector[int] out = vector[int](v.size())
+#     inclusive_scan(seq, v.begin(), v.end(), out.begin(), add_integers, init)
+#     return out
+
+def test_transform_inclusive_scan(vector[int] v):
+    """
+    Test transform inclusive_scan
+    >>> test_transform_inclusive_scan([1, 2, 3, 4])
+    [2, 6, 12, 20]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    transform_inclusive_scan(v.begin(), v.end(), out.begin(), add_integers, multiply_with_2)
+    return out
+
+# def test_transform_inclusive_scan_with_execpolicy(vector[int] v):
+#     """
+#     Test transform inclusive_scan with a execution policy
+#     >>> test_transform_inclusive_scan_with_execpolicy([1, 2, 3, 4])
+#     [2, 6, 12, 20 ]
+#     """
+#     cdef vector[int] out
+#     transform_inclusive_scan(seq, v.begin(), v.end(), out.begin(), add_integers, multiply_with_2)
+#     return out
+
+def test_transform_inclusive_scan_with_init(vector[int] v, int init):
+    """
+    Test transform inclusive_scan with an initial value
+    >>> test_transform_inclusive_scan_with_init([1, 2, 3, 4], 0)
+    [2, 6, 12, 20]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    transform_inclusive_scan(v.begin(), v.end(), out.begin(), add_integers, multiply_with_2, init)
+    return out
+
+def test_transform_inclusive_scan_with_execpolicy_and_init(vector[int] v, int init):
+    """
+    Test transform inclusive_scan with an initial value
+    >>> test_transform_inclusive_scan_with_execpolicy_and_init([1, 2, 3, 4], 0)
+    [2, 6, 12, 20]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    transform_inclusive_scan(seq, v.begin(), v.end(), out.begin(), add_integers, multiply_with_2, init)
+    return out
+
+def test_exclusive_scan(vector[int] v, int init):
+    """
+    Test exclusive_scan
+    >>> test_exclusive_scan([1, 2, 3, 4], 0)
+    [0, 1, 3, 6]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    exclusive_scan(v.begin(), v.end(), out.begin(), init)
+    return out
+
+# def test_exclusive_scan_with_execpolicy(vector[int] v, int init):
+#     """
+#     Test exclusive_scan with a execution policy
+#     >>> test_exclusive_scan_with_execpolicy([1, 2, 3, 4], 0)
+#     [0, 1, 3, 6]
+#     """
+#     cdef vector[int] out
+#     exclusive_scan(seq, v.begin(), v.end(), out.begin(), init)
+#     return out
+
+def test_exclusive_scan_with_bin_op(vector[int] v, int init):
+    """
+    Test exclusive_scan with a binary operation
+    >>> test_exclusive_scan_with_bin_op([1, 2, 3, 4], 0)
+    [0, 1, 3, 6]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    exclusive_scan(v.begin(), v.end(), out.begin(), init, add_integers)
+    return out
+
+def test_exclusive_scan_with_execpolicy_and_bin_op(vector[int] v, int init):
+    """
+    Test exclusive_scan with a execution policy and a binary operation
+    >>> test_exclusive_scan_with_execpolicy_and_bin_op([1, 2, 3, 4], 0)
+    [0, 1, 3, 6]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    exclusive_scan(seq, v.begin(), v.end(), out.begin(), init, add_integers)
+    return out
+
+
+def test_transform_exclusive_scan_with_execpolicy(vector[int] v, int init):
+    """
+    Test transform_exclusive_scan
+    >>> test_transform_exclusive_scan_with_execpolicy([1, 2, 3, 4], 0)
+    [0, 2, 6, 12]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    transform_exclusive_scan(seq, v.begin(), v.end(), out.begin(), init, add_integers, multiply_with_2)
+    return out
+
+def test_transform_exclusive_scan_with_execpolicy(vector[int] v, int init):
+    """
+    Test transform_exclusive_scan with a execution policy
+    >>> test_transform_exclusive_scan_with_execpolicy([1, 2, 3, 4], 0)
+    [0, 2, 6, 12]
+    """
+    cdef vector[int] out = vector[int](v.size())
+    transform_exclusive_scan(seq, v.begin(), v.end(), out.begin(), init, add_integers, multiply_with_2)
+    return out
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_numeric_ops.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_numeric_ops.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_numeric_ops.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_numeric_ops.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,147 @@
+# mode: run
+# tag: cpp, werror, cpp11
+
+from libcpp.numeric cimport inner_product, iota, accumulate, adjacent_difference, partial_sum
+from libcpp.vector cimport vector
+from libcpp cimport bool
+
+# Subtracts two integers.
+cdef int subtract_integers(int lhs, int rhs):
+    return lhs - rhs
+
+# Adds two integers.
+cdef int add_integers(int lhs, int rhs):
+    return lhs + rhs
+
+# Multiplies two integers.
+cdef int multiply_integers(int lhs, int rhs):
+    return lhs * rhs
+
+# Determines equality for two integers.
+# If lhs == rhs, returns true. Returns false otherwise.
+cdef bool is_equal(int lhs, int rhs):
+    return lhs == rhs
+
+def test_inner_product(vector[int] v1, vector[int] v2, int init):
+    """
+    Test inner_product with integer values.
+    >>> test_inner_product([1, 2, 3], [1, 2, 3], 1)
+    15
+    """
+    return inner_product(v1.begin(), v1.end(), v2.begin(), init)
+
+
+def test_inner_product_with_zero(vector[int] v1, vector[int] v2, int init):
+    """
+    Test inner_product with a zero value in the container.
+    >>> test_inner_product_with_zero([1, 2, 0], [1, 1, 1], 0)
+    3
+    """
+    return inner_product(v1.begin(), v1.end(), v2.begin(), init)
+
+def test_inner_product_with_bin_op(vector[int] v1, vector[int] v2, int init):
+    """
+    Test inner_product with two binary operations. In this case,
+    Looks at number of pairwise matches between v1 and v2.
+    [5, 1, 2, 3, 4]
+    [5, 4, 2, 3, 1]
+    There are 3 matches (5, 2, 3). So, 1 + 1 + 1 = 3.
+
+    >>> test_inner_product_with_bin_op([5, 1, 2, 3, 4], [5, 4, 2, 3, 1], 0)
+    3
+    """
+    return inner_product(v1.begin(), v1.end(), v2.begin(), init, add_integers, is_equal)
+
+def test_iota(vector[int] v, int value):
+    """
+    Test iota with beginning value of 0.
+    >>> test_iota(range(6), 0)
+    [0, 1, 2, 3, 4, 5]
+    """
+    iota(v.begin(), v.end(), value)
+    return v
+
+def test_iota_negative_init(vector[int] v, int value):
+    """
+    Test iota with a negative beginning value.
+    >>> test_iota_negative_init(range(7), -4)
+    [-4, -3, -2, -1, 0, 1, 2]
+    """
+    iota(v.begin(), v.end(), value)
+    return v
+
+def test_accumulate(vector[int] v, int init):
+    """
+    Test accumulate.
+     0 + 1 = 1
+     1 + 2 = 3
+     3 + 3 = 6
+    >>> test_accumulate([1, 2, 3], 0)
+    6
+    """
+    return accumulate(v.begin(), v.end(), init)
+
+def test_accumulate_with_subtraction(vector[int] v, int init):
+    """
+    Test accumulate with subtraction. Note that accumulate is a fold-left operation.
+     0 - 1 = -1
+    -1 - 2 = -3
+    -3 - 3 = -6
+    >>> test_accumulate_with_subtraction([1, 2, 3], 0)
+    -6
+    """
+    return accumulate(v.begin(), v.end(), init, subtract_integers)
+
+def test_adjacent_difference(vector[int] v):
+    """
+    Test adjacent_difference with integer values.
+    2 - 0,   -> 2
+    4 - 2,   -> 2
+    6 - 4,   -> 2
+    8 - 6,   -> 2
+    10 - 8,  -> 2
+    12 - 10  -> 2
+    >>> test_adjacent_difference([2, 4, 6, 8, 10, 12])
+    [2, 2, 2, 2, 2, 2]
+    """
+    adjacent_difference(v.begin(), v.end(), v.begin())
+    return v
+
+def test_adjacent_difference_with_bin_op(vector[int] v):
+    """
+    Test adjacent_difference with a binary operation.
+    0 + 1 -> 1
+    1 + 2 -> 3
+    2 + 4 -> 6
+    4 + 5 -> 9
+    5 + 6 -> 11
+    >>> test_adjacent_difference_with_bin_op([1, 2, 4, 5, 6])
+    [1, 3, 6, 9, 11]
+    """
+    adjacent_difference(v.begin(), v.end(), v.begin(), add_integers)
+    return v
+
+def test_partial_sum(vector[int] v):
+    """
+    Test partial_sum with integer values.
+    2 + 0   -> 2
+    2 + 2   -> 4
+    4 + 2   -> 6
+    6 + 2   -> 8
+    8 + 2   -> 10
+    10 + 2  -> 12
+    >>> test_partial_sum([2, 2, 2, 2, 2, 2])
+    [2, 4, 6, 8, 10, 12]
+    """
+    partial_sum(v.begin(), v.end(), v.begin())
+    return v
+
+def test_partial_sum_with_bin_op(vector[int] v):
+    """
+    Test partial_sum with a binary operation.
+    Using multiply_integers, partial_sum will calculate the first 5 powers of 2.
+    >>> test_partial_sum_with_bin_op([2, 2, 2, 2, 2])
+    [2, 4, 8, 16, 32]
+    """
+    partial_sum(v.begin(), v.end(), v.begin(), multiply_integers)
+    return v
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_optional.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_optional.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_optional.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_optional.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,72 @@
+# ticket: 3293
+# mode: run
+# tag: cpp, cpp17, werror
+
+from cython.operator cimport dereference as deref
+from libcpp.optional cimport optional, nullopt, make_optional
+from libcpp.string cimport string
+from libcpp.pair cimport pair
+
+def simple_test():
+    """
+    >>> simple_test()
+    """
+    cdef optional[int] o
+    assert(not o.has_value())
+    o = 5
+    assert(o.has_value())
+    assert(o.value()==5)
+    o.reset()
+    assert(not o.has_value())
+
+def operator_test():
+    """
+    >>> operator_test()
+    """
+    cdef optional[int] o1, o2
+
+    # operator bool
+    assert(not o1)
+    o1 = 5
+    assert(o1)
+
+    # operator *
+    assert(deref(o1) == 5)
+
+    # operator =,==,!=,>,<,>=,<=
+    assert(not o1 == o2)
+    assert(o1 != o2)
+    o2 = o1
+    assert(o1 == o2)
+    assert(not o1 > o2)
+    assert(not o1 < o2)
+    assert(o1 >= o2)
+    assert(o1 <= o2)
+
+    # operators =,== for other types (all related operators are identical)
+    o1 = 6
+    assert(o1 == 6)
+    o2 = nullopt
+    assert(o2 == nullopt)
+
+def misc_methods_test():
+    """
+    >>> misc_methods_test()
+    """
+
+    # make_optional
+    cdef optional[int] o
+    o = make_optional[int](5)
+    assert(o == 5)
+
+    # swap
+    o.swap(optional[int](6))
+    assert(o == 6)
+
+    # emplace
+    cdef optional[pair[int,int]] op
+    cdef pair[int,int]* val_ptr = &op.emplace(1,2)
+    assert(op.has_value())
+    assert(op.value() == pair[int,int](1,2))
+    assert(&op.value() == val_ptr) # check returned reference
+    
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_set.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_set.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_set.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_set.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,137 @@
+# mode: run
+# tag: cpp, cpp11
+
+# cython: language_level=3
+
+from libcpp.set cimport set
+from libcpp.unordered_set cimport unordered_set
+from libcpp.utility cimport pair
+
+def test_set_insert(vals):
+    """
+    >>> test_set_insert([1,2,2,3, -1])
+    [-1, 1, 2, 3]
+    """
+    cdef set[int] s = set[int]()
+    cdef pair[set[int].iterator, bint] ret
+    for v in vals:
+        ret = s.insert(v)
+    return [item for item in s]
+
+def test_set_insert_it(vals):
+    """
+    >>> test_set_insert_it([1,2,2,3, -1])
+    [-1, 1, 2, 3]
+    """
+    cdef unordered_set[int] us = unordered_set[int]()
+    cdef set[int] s = set[int]()
+    for v in vals:
+        us.insert(v)
+    s.insert(us.begin(), us.end())
+    return [item for item in s]
+
+def test_set_count(vals, to_find):
+    """
+    >>> test_set_count([1,2,2,3, -1], 1)
+    1
+    >>> test_set_count([1,2,2,3, -1], 2)
+    1
+    """
+    cdef set[int] s = set[int]()
+    for v in vals:
+        s.insert(v)
+    return s.count(to_find)
+
+def test_set_erase(vals, int to_remove):
+    """
+    >>> test_set_erase([1,2,2,3, -1], 1)
+    [-1, 2, 3]
+    >>> test_set_erase([1,2,2,3, -1], 2)
+    [-1, 1, 3]
+    """
+    cdef set[int] s = set[int]()
+    cdef size_t ret
+    for v in vals:
+        s.insert(v)
+    ret = s.erase(to_remove)
+    return [item for item in s]
+
+def test_set_find_erase(vals, to_remove):
+    """
+    >>> test_set_find_erase([1,2,2,3, -1], 1)
+    [-1, 2, 3]
+    >>> test_set_find_erase([1,2,2,3, -1], 2)
+    [-1, 1, 3]
+    """
+    cdef set[int] s = set[int]()
+    cdef set[int].iterator it
+    for v in vals:
+        s.insert(v)
+    it = s.find(to_remove)
+    it = s.erase(it)
+    return [item for item in s]
+
+
+def test_unordered_set_insert(vals):
+    """
+    >>> test_unordered_set_insert([1,2,2,3, -1])
+    [-1, 1, 2, 3]
+    """
+    cdef unordered_set[int] us = unordered_set[int]()
+    cdef pair[unordered_set[int].iterator, bint] ret
+    for v in vals:
+        ret = us.insert(v)
+    return sorted([item for item in us])
+
+def test_unordered_set_insert_it(vals):
+    """
+    >>> test_unordered_set_insert_it([1,2,2,3, -1])
+    [-1, 1, 2, 3]
+    """
+    cdef set[int] s = set[int]()
+    cdef unordered_set[int] us = unordered_set[int]()
+    for v in vals:
+        s.insert(v)
+    us.insert(s.begin(), s.end())
+    return sorted([item for item in us])
+
+def test_unordered_set_count(vals, to_find):
+    """
+    >>> test_unordered_set_count([1,2,2,3, -1], 1)
+    1
+    >>> test_unordered_set_count([1,2,2,3, -1], 2)
+    1
+    """
+    cdef unordered_set[int] us = unordered_set[int]()
+    for v in vals:
+        us.insert(v)
+    return us.count(to_find)
+
+def test_unordered_set_erase(vals, int to_remove):
+    """
+    >>> test_unordered_set_erase([1,2,2,3, -1], 1)
+    [-1, 2, 3]
+    >>> test_unordered_set_erase([1,2,2,3, -1], 2)
+    [-1, 1, 3]
+    """
+    cdef unordered_set[int] us = unordered_set[int]()
+    cdef size_t ret
+    for v in vals:
+        us.insert(v)
+    ret = us.erase(to_remove)
+    return sorted([item for item in us])
+
+def test_unordered_set_find_erase(vals, to_remove):
+    """
+    >>> test_unordered_set_find_erase([1,2,2,3, -1], 1)
+    [-1, 2, 3]
+    >>> test_unordered_set_find_erase([1,2,2,3, -1], 2)
+    [-1, 1, 3]
+    """
+    cdef unordered_set[int] us = unordered_set[int]()
+    cdef unordered_set[int].iterator it
+    for v in vals:
+        us.insert(v)
+    it = us.find(to_remove)
+    it = us.erase(it)
+    return sorted([item for item in us])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_ascii_auto_encoding.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_ascii_auto_encoding.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_ascii_auto_encoding.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_ascii_auto_encoding.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,9 +2,8 @@
 # tag: cpp, werror
 # cython: c_string_encoding=ascii, c_string_type=unicode
 
-cimport cython
-
 from libcpp.string cimport string
+from libcpp.vector cimport vector
 
 b_asdf = b'asdf'
 s_asdf = 'asdf'
@@ -148,3 +147,17 @@
     cdef string s = a
     assert s.length() == len(a), "%d != %d" % (s.length(), len(a))
     return s
+
+
+def test_vector_of_strings(*strings):
+    """
+    >>> results = test_vector_of_strings(b_asdf, u_asdf)
+    >>> results == [u_asdf, u_asdf] or results
+    True
+    >>> type(results[0]) is type(u_asdf) or type(results[0])
+    True
+    >>> type(results[1]) is type(u_asdf) or type(results[1])
+    True
+    """
+    cdef vector[string] v = strings
+    return v
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_cpp11.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_cpp11.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_cpp11.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_cpp11.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,15 @@
+# mode: run
+# tag: cpp, werror, cpp11
+
+from libcpp.string cimport string
+
+b_asdf = b'asdf'
+
+def test_element_access(char *py_str):
+    """
+    >>> test_element_access(b_asdf)
+    ('a', 'f')
+    """
+    cdef string s
+    s = string(py_str)
+    return chr(s.front()), chr(s.back())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,14 +1,48 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, warnings
 
 cimport cython
 
-from libcpp.string cimport string
+from libcpp.string cimport string, npos, to_string, stoi, stof
 
 b_asdf = b'asdf'
 b_asdg = b'asdg'
 b_s = b's'
 
+
+cdef int compare_to_asdf_ref(string& s) except -999:
+    return s.compare(b"asdf")
+
+def test_coerced_literal_ref():
+    """
+    >>> test_coerced_literal_ref()
+    0
+    """
+    return compare_to_asdf_ref("asdf")
+
+
+cdef int compare_to_asdf_const_ref(const string& s) except -999:
+    return s.compare(b"asdf")
+
+def test_coerced_literal_const_ref():
+    """
+    >>> test_coerced_literal_const_ref()
+    0
+    """
+    return compare_to_asdf_const_ref("asdf")
+
+
+cdef int compare_to_asdf_const(const string s) except -999:
+    return s.compare(b"asdf")
+
+def test_coerced_literal_const():
+    """
+    >>> test_coerced_literal_const()
+    0
+    """
+    return compare_to_asdf_const("asdf")
+
+
 def test_conversion(py_obj):
     """
     >>> test_conversion(b_asdf) == b_asdf or test_conversion(b_asdf)
@@ -68,6 +102,15 @@
     s.push_back(ord('s'))
     return s.c_str()
 
+def test_pop_back(char *a):
+    """
+    >>> test_pop_back(b'abc') == b'ab' or test_pop_back(b'abc')
+    True
+    """
+    cdef string s = string(a)
+    s.pop_back()
+    return s
+
 def test_insert(char *a, char *b, int i):
     """
     >>> test_insert('AAAA'.encode('ASCII'), 'BBBB'.encode('ASCII'), 2) == 'AABBBBAA'.encode('ASCII')
@@ -99,6 +142,17 @@
     cdef size_t i = s.find(t)
     return i
 
+def test_npos(char *a, char *b):
+    """
+    >>> test_npos(b'abc', b'x')
+    True
+    >>> test_npos(b'abc', b'a')
+    False
+    """
+    cdef string s = string(a)
+    cdef string st = string(b)
+    return s.find(st) == npos
+
 def test_clear():
     """
     >>> test_clear() == ''.encode('ASCII')
@@ -108,6 +162,18 @@
     s.clear()
     return s.c_str()
 
+def test_erase(char *a, size_t pos=0, size_t count=npos):
+    """
+    >>> test_erase(b'abc') == b'' or test_erase(b'abc')
+    True
+    >>> test_erase(b'abc', 1) == b'a' or test_erase(b'abc', 1)
+    True
+    >>> test_erase(b'abc', 1, 1) == b'ac' or test_erase(b'abc', 1, 1)
+    True
+    """
+    cdef string s = string(a)
+    return s.erase(pos, count)
+
 def test_assign(char *a):
     """
     >>> test_assign(b_asdf) == 'ggg'.encode('ASCII')
@@ -310,3 +376,78 @@
     []
     """
     return [c for c in s]
+
+def test_to_string(x):
+    """
+    >>> print(test_to_string(5))
+    si=5 sl=5 ss=5 sss=5
+    >>> print(test_to_string(-5))
+    si=-5 sl=-5 ss=5 sss=-5
+    """
+    si = to_string(x).decode('ascii')
+    sl = to_string(x).decode('ascii')
+    ss = to_string(abs(x)).decode('ascii')
+    sss = to_string(x).decode('ascii')
+    return f"si={si} sl={sl} ss={ss} sss={sss}"
+
+def test_stoi(char *a):
+    """
+    >>> test_stoi(b'5')
+    5
+    """
+    cdef string s = string(a)
+    return stoi(s)
+
+def test_stof(char *a):
+    """
+    >>> test_stof(b'5.5')
+    5.5
+    """
+    cdef string s = string(a)
+    return stof(s)
+
+def test_to_string(x):
+    """
+    >>> print(test_to_string(5))
+    si=5 sl=5 ss=5 sss=5
+    >>> print(test_to_string(-5))
+    si=-5 sl=-5 ss=5 sss=-5
+    """
+    si = to_string(x).decode('ascii')
+    sl = to_string(x).decode('ascii')
+    ss = to_string(abs(x)).decode('ascii')
+    sss = to_string(x).decode('ascii')
+    return f"si={si} sl={sl} ss={ss} sss={sss}"
+
+
+def test_stoi(char *a):
+    """
+    >>> test_stoi(b'5')
+    5
+    """
+    cdef string s = string(a)
+    return stoi(s)
+
+
+def test_stof(char *a):
+    """
+    >>> test_stof(b'5.5')
+    5.5
+    """
+    cdef string s = string(a)
+    return stof(s)
+
+
+def test_swap():
+    """
+    >>> test_swap()
+    """
+    cdef string s1 = b_asdf, s_asdf = b_asdf
+    cdef string s2 = b_asdg, s_asdg = b_asdg
+    s1.swap(s2)
+    assert s1 == s_asdg and s2 == s_asdf
+
+
+_WARNINGS = """
+21:31: Cannot pass Python object as C++ data structure reference (string &), will pass by copy.
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_utf8_auto_encoding.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_utf8_auto_encoding.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_string_utf8_auto_encoding.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_string_utf8_auto_encoding.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,150 @@
+# mode: run
+# tag: cpp, werror
+# cython: c_string_encoding=utf-8, c_string_type=unicode
+
+cimport cython
+
+from libcpp.string cimport string
+
+b_asdf = b'asdf'
+s_asdf = 'asdf'
+u_asdf = u'asdf'
+u_s = u's'
+
+
+def test_conversion(py_obj):
+    """
+    >>> test_conversion(b_asdf) == u_asdf or test_conversion(b_asdf)
+    True
+    >>> test_conversion(u_asdf) == u_asdf or test_conversion(u_asdf)
+    True
+    >>> test_conversion(123)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: expected ..., int found
+    """
+    cdef string s = py_obj
+    assert len(py_obj) == s.length(), '%d != %d' % (len(py_obj), s.length())
+    return s
+
+
+def test_empty(py_obj):
+    """
+    >>> test_empty('')
+    True
+    >>> test_empty('abc')
+    False
+    >>> test_empty(u_asdf[:0])
+    True
+    >>> test_empty(u_asdf)
+    False
+    """
+    cdef string a = py_obj
+    return a.empty()
+
+
+def test_push_back(a):
+    """
+    >>> test_push_back(b_asdf) == u_asdf + u_s
+    True
+    >>> test_push_back(u_asdf) == u_asdf + u_s
+    True
+    """
+    cdef string s = a
+    s.push_back(ord('s'))
+    return s
+
+
+def test_clear(a):
+    """
+    >>> test_clear(u_asdf) == u_s[:0]
+    True
+    >>> test_clear(b_asdf) == u_s[:0]
+    True
+    """
+    cdef string s = a
+    s.clear()
+    return s
+
+
+def test_assign(char *a):
+    """
+    >>> test_assign(b_asdf) == 'ggg'
+    True
+    """
+    cdef string s = string(a)
+    s.assign("ggg")
+    return s.c_str()
+
+
+def test_bytes_cast(a):
+    """
+    >>> b = test_bytes_cast(b'abc')
+    >>> isinstance(b, bytes)
+    True
+    >>> print(b.decode('ascii'))
+    abc
+    >>> b = test_bytes_cast(b'abc\\xe4\\xfc')
+    >>> isinstance(b, bytes)
+    True
+    >>> len(b)
+    5
+    >>> print(b[:3].decode('ascii'))
+    abc
+    >>> print(ord(b[3:4]))
+    228
+    >>> print(ord(b[4:5]))
+    252
+    """
+    cdef string s = a
+    assert s.length() == len(a), "%d != %d" % (s.length(), len(a))
+    return s
+
+
+def test_bytearray_cast(a):
+    """
+    >>> b = test_bytearray_cast(b'abc')
+    >>> isinstance(b, bytearray)
+    True
+    >>> print(b.decode('ascii'))
+    abc
+    >>> b = test_bytearray_cast(b'abc\\xe4\\xfc')
+    >>> isinstance(b, bytearray)
+    True
+    >>> len(b)
+    5
+    >>> print(b[:3].decode('ascii'))
+    abc
+    >>> print(ord(b[3:4]))
+    228
+    >>> print(ord(b[4:5]))
+    252
+    """
+    cdef string s = a
+    assert s.length() == len(a), "%d != %d" % (s.length(), len(a))
+    return s
+
+
+def test_unicode_cast(a):
+    """
+    >>> u = test_unicode_cast(b'abc')
+    >>> type(u) is type(u_asdf) or type(u)
+    True
+    >>> print(u)
+    abc
+    """
+    cdef string s = a
+    assert s.length() == len(a), "%d != %d" % (s.length(), len(a))
+    return s
+
+
+def test_str_cast(a):
+    """
+    >>> s = test_str_cast(b'abc')
+    >>> type(s) is type(s_asdf) or type(s)
+    True
+    >>> print(s)
+    abc
+    """
+    cdef string s = a
+    assert s.length() == len(a), "%d != %d" % (s.length(), len(a))
+    return s
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_vector.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_vector.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_stl_vector.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_stl_vector.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator cimport dereference as d
 from cython.operator cimport preincrement as incr
@@ -177,3 +177,44 @@
     v[0] = True
     v[1] = False
     assert v == [True, False, True, True, True]
+
+ctypedef vector[cbool] vector_bool
+ctypedef vector[int] vector_int
+
+def test_typedef_vector(L):
+    """
+    >>> test_typedef_vector([0, 1, True])
+    ([0, 1, 1, 0, 1, 1], 0, [False, True, True, False, True, True], False)
+    """
+    cdef vector_int vi = L
+    cdef vector_int vi2 = vi
+    vi.insert(vi.begin(), vi2.begin(), vi2.end())
+
+    cdef vector_bool vb = L
+    cdef vector_bool vb2 = vb
+    vb.insert(vb.begin(), vb2.begin(), vb2.end())
+
+    return vi, vi.at(0), vb, vb.at(0)
+
+
+def test_insert():
+    """
+    >>> test_insert()
+    """
+    cdef vector[int] v
+    cdef vector[int].size_type count = 5
+    cdef int value = 0
+
+    v.insert(v.end(), count, value)
+
+    assert v.size() == count
+    for element in v:
+        assert element == value, '%s != %s' % (element, count)
+
+
+#  Tests GitHub issue #1788.
+cdef cppclass MyVector[T](vector):
+    pass
+
+cdef cppclass Ints(MyVector[int]):
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_functions_helper.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_functions_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_functions_helper.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_functions_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -20,6 +20,29 @@
         std::pair method(T a, U b) {
             return std::pair(a, b);
         }
+        template 
+        U part_method(std::pair p) {
+            return p.second;
+        }
+        template 
+        U part_method_ref(const std::pair& p) {
+            return p.second;
+        }
+
+        int overloaded(double d) {
+            return (int) d;
+        }
+        T overloaded(std::pair p) {
+            return p.first;
+        }
+        template 
+        U overloaded(std::vector v) {
+            return v[0];
+        }
+        template 
+        U overloaded(char* c, std::vector v) {
+            return v[0];
+        }
 };
 
 template 
@@ -31,3 +54,13 @@
 std::pair pair_arg(std::pair a) {
     return a;
 }
+
+template 
+T* pointer_param(T* param) {
+    return param;
+}
+
+class double_pair : public std::pair {
+  public:
+    double_pair(double x, double y) : std::pair(x, y) { };
+};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_functions.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_functions.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_functions.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_functions.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,9 @@
-# tag: cpp
+# mode: run
+# tag: cpp, warnings, no-cpp-locals
 
+cimport cython
 from libcpp.pair cimport pair
+from libcpp.vector cimport vector
 
 cdef extern from "cpp_template_functions_helper.h":
     cdef T no_arg[T]()
@@ -8,8 +11,17 @@
     cdef pair[T, U] two_params[T, U](T, U)
     cdef cppclass A[T]:
         pair[T, U] method[U](T, U)
+        U part_method[U](pair[T, U])
+        U part_method_ref[U](pair[T, U]&)
+        int overloaded(double x)
+        T overloaded(pair[T, T])
+        U overloaded[U](vector[U])
+        X overloaded[X](char* s, vector[X])
     cdef T nested_deduction[T](const T*)
     pair[T, U] pair_arg[T, U](pair[T, U] a)
+    cdef T* pointer_param[T](T*)
+    cdef cppclass double_pair(pair[double, double]):
+        double_pair(double, double)
 
 def test_no_arg():
     """
@@ -35,13 +47,27 @@
 def test_method(int x, int y):
     """
     >>> test_method(5, 10)
-    ((5, 10.0), (5.0, 10))
+    ((5, 10.0), (5.0, 10), (5, 10), (5.0, 10))
     """
     cdef A[int] a_int
     cdef A[double] a_double
-    return a_int.method[float](x, y), a_double.method[int](x, y)
+    return (a_int.method[float](x, y), a_double.method[int](x, y),
+        a_int.method(x, y), a_double.method(x, y))
 #    return a_int.method[double](x, y), a_double.method[int](x, y)
 
+def test_part_method(int x, int y):
+    """
+    >>> test_part_method(5, 10)
+    (10.0, 10, 10.0)
+    """
+    cdef A[int] a_int
+    cdef pair[int, double] p_int = (x, y)
+    cdef A[double] a_double
+    cdef pair[double, int] p_double = (x, y)
+    return (a_int.part_method(p_int),
+        a_double.part_method(p_double),
+        a_double.part_method_ref(double_pair(x, y)))
+
 def test_simple_deduction(int x, double y):
     """
     >>> test_simple_deduction(1, 2)
@@ -63,3 +89,33 @@
     """
     return pair_arg(x)
 
+def test_deduce_through_pointers(int k):
+    """
+    >>> test_deduce_through_pointers(5)
+    (5, 5.0)
+    """
+    cdef double x = k
+    return pointer_param(&k)[0], pointer_param(&x)[0]
+
+def test_inference(int k):
+    """
+    >>> test_inference(27)
+    27
+    """
+    res = one_param(&k)
+    assert cython.typeof(res) == 'int *', cython.typeof(res)
+    return res[0]
+
+def test_overload_GH1583():
+    """
+    >>> test_overload_GH1583()
+    """
+    cdef A[int] a
+    assert a.overloaded(1.5) == 1
+    cdef pair[int, int] p = (2, 3)
+    assert a.overloaded(p) == 2
+    cdef vector[double] v = [0.25, 0.125]
+    assert a.overloaded(v) == 0.25
+    assert a.overloaded("s", v) == 0.25
+    # GH Issue #1584
+    # assert a.overloaded[double](v) == 0.25
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_ref_args.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_ref_args.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_ref_args.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_ref_args.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 from libcpp.vector cimport vector
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_templates_helper.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_templates_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_templates_helper.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_templates_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -44,3 +44,9 @@
 public:
     static T half(T value) { return value / 2; }
 };
+
+template 
+class BinaryAnd {
+public:
+    static T1 call(T1 x, T2 y) { return x & y; }
+};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_templates.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_templates.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_templates.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_templates.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,6 @@
 # tag: cpp
 
+cimport cython
 from cython.operator import dereference as deref
 
 cdef extern from "cpp_templates_helper.h":
@@ -130,6 +131,19 @@
     finally:
         del w
 
+def test_typeof(double x):
+    """
+    >>> test_func_ptr(3)
+    9.0
+    >>> test_func_ptr(-1.5)
+    2.25
+    """
+    try:
+        w = new Wrap[cython.typeof(&f)](&f)
+        return w.get()(x)
+    finally:
+        del w
+
 def test_cast_template_pointer():
     """
     >>> test_cast_template_pointer()
@@ -148,3 +162,16 @@
     (1, 1.5)
     """
     return Div[int].half(x), Div[double].half(x)
+
+def test_pure_syntax(int i):
+    """
+    >>> test_ptr(3)
+    3
+    >>> test_ptr(5)
+    5
+    """
+    try:
+        w = new Wrap[cython.pointer(int)](&i)
+        return deref(w.get())
+    finally:
+        del w
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_subclasses.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_subclasses.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_template_subclasses.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_template_subclasses.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: cpp, werror
+# tag: cpp, werror, no-cpp-locals
 
 from cython.operator import dereference as deref
 from libcpp.pair cimport pair
@@ -115,3 +115,53 @@
     finally:
         del e
 
+
+cdef public pair[int, double] public_return_pair(a, b) except *:
+  return pair[int, double](a, b)
+
+def test_GH1599(a, b):
+  """
+  >>> test_GH1599(1, 2)
+  (1, 2.0)
+  """
+  return public_return_pair(a, b)
+
+
+# Related to GH Issue #1852.
+
+cdef cppclass Callback[T]:#(UntypedCallback):
+    pass
+
+cdef cppclass MyClass[O]:
+    void Invoke(Callback[O]*)
+
+cdef cppclass MySubclass[T](MyClass[T]):
+    void Invoke(Callback[T]* callback):
+      pass
+
+
+cdef cppclass Getter[T]:
+    T get(bint fire) except *:
+        if fire:
+            raise RuntimeError
+        else:
+           raise NotImplementedError
+
+cdef cppclass GetInt(Getter[int]):
+    int get(bint fire) except *:
+        if fire:
+            raise RuntimeError
+        else:
+            return 389
+
+def test_subclass_exception_values(bint fire):
+    """
+    >>> test_subclass_exception_values(False)
+    389
+    >>> test_subclass_exception_values(True)
+    Traceback (most recent call last):
+    ...
+    RuntimeError
+    """
+    cdef GetInt getter
+    return getter.get(fire)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_type_inference.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpp_type_inference.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_type_inference.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_type_inference.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -12,6 +12,11 @@
     cdef cppclass Square(Shape):
         Square(int)
 
+    cdef cppclass Empty(Shape):
+        Empty()
+
+    cdef Empty make_Empty "shapes::Empty"()
+
 from cython cimport typeof
 
 from cython.operator cimport dereference as d
@@ -48,3 +53,35 @@
         ptr = new Square(size)
     print typeof(ptr)
     del ptr
+
+def test_stack_allocated(bint b=True):
+    """
+    >>> test_stack_allocated()
+    """
+    e1 = Empty()
+    e2 = Empty()
+    e = e1 if b else e2
+    assert typeof(e1) == "Empty", typeof(e1)
+    assert typeof(e2) == "Empty", typeof(e2)
+    assert typeof(e) == "Empty", typeof(e)
+
+cdef extern from *:
+    """
+    template 
+    struct MyTemplate {};
+    """
+    cdef cppclass MyTemplate[T]:
+        MyTemplate()
+
+def test_template_types():
+    """
+    >>> test_template_types()
+    """
+    t_stack = MyTemplate[int]()
+    assert typeof(t_stack) == "MyTemplate[int]", typeof(t_stack)
+
+    t_ptr = new MyTemplate[double]()
+    try:
+        assert typeof(t_ptr) == "MyTemplate[double] *", typeof(t_ptr)
+    finally:
+        del t_ptr
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpp_unordered_map_helper.h cython-0.20.1+1~202203241016-9537/tests/run/cpp_unordered_map_helper.h
--- cython-0.20.1+1~201611251650-6686/tests/run/cpp_unordered_map_helper.h	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpp_unordered_map_helper.h	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,13 @@
+#include 
+#include 
+
+struct IntVectorHash {
+  size_t operator()(const std::vector& v) const {
+    std::hash hasher;
+    size_t seed = 0;
+    for (int i : v) {
+      seed ^= hasher(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+    }
+    return seed;
+  }
+};
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpython_capi_py35.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpython_capi_py35.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpython_capi_py35.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpython_capi_py35.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,63 @@
+# mode: run
+# tag: c-api
+
+# PyMem_RawMalloc tests that need to be disabled for Python < 3.5
+# (some of these would work of Python 3.4, but it's easier to disable
+# them in one place)
+
+from cpython cimport mem
+
+cdef short _assert_calloc(short* s, int n) except -1 with gil:
+    """Assert array ``s`` of length ``n`` is zero and return 3."""
+    assert not s[0] and not s[n - 1]
+    s[0] += 1
+    s[n - 1] += 3
+    for i in range(1, n - 1):
+        assert not s[i]
+    return s[n - 1]
+
+def test_pycalloc():
+    """
+    >>> test_pycalloc()
+    3
+    """
+    cdef short* s =  mem.PyMem_Calloc(10, sizeof(short))
+    if not s:
+        raise MemoryError()
+    try:
+        return _assert_calloc(s, 10)
+    finally:
+        mem.PyMem_Free(s)
+
+def test_pymalloc_raw():
+    """
+    >>> test_pymalloc_raw()
+    3
+    """
+    cdef short i
+    cdef short* s
+    cdef char* m
+    cdef char* m2 = NULL
+    with nogil:
+        s =  mem.PyMem_RawCalloc(10, sizeof(short))
+        if not s:
+            raise MemoryError()
+        try:
+            i = _assert_calloc(s, 10)
+        finally:
+            mem.PyMem_RawFree(s)
+        m =  mem.PyMem_RawMalloc(20)
+        if not m:
+            raise MemoryError()
+        try:
+            m[0] = 1
+            m[1] = 2
+            m[2] = i
+            m2 =  mem.PyMem_RawRealloc(m, 10)
+            if m2:
+                m = m2
+            retval = m[2]
+        finally:
+            mem.PyMem_RawFree(m)
+    assert m2
+    return retval
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cpython_capi.pyx cython-0.20.1+1~202203241016-9537/tests/run/cpython_capi.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cpython_capi.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cpython_capi.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,39 @@
+# mode: run
+# tag: c-api
+
+from cpython cimport mem
+from cpython.pystate cimport PyGILState_Ensure, PyGILState_Release, PyGILState_STATE
+
+
+def test_pymalloc():
+    """
+    >>> test_pymalloc()
+    3
+    """
+    cdef char* m2
+    cdef char* m =  mem.PyMem_Malloc(20)
+    assert m
+    try:
+        m[0] = 1
+        m[1] = 2
+        m[2] = 3
+        m2 =  mem.PyMem_Realloc(m, 10)
+        assert m2
+        m = m2
+        return m[2]
+    finally:
+        mem.PyMem_Free(m)
+
+
+def test_gilstate():
+    """
+    >>> test_gilstate()
+    'ok'
+    """
+
+    # cython used to have invalid definition for PyGILState_STATE, which was
+    # making the following code fail to compile
+    cdef PyGILState_STATE gstate = PyGILState_Ensure()
+    # TODO assert that GIL is taken
+    PyGILState_Release(gstate)
+    return 'ok'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/crashT245.pyx cython-0.20.1+1~202203241016-9537/tests/run/crashT245.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/crashT245.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/crashT245.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 245
+# ticket: t245
 
 cimport crashT245_pxd
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cstringmul.pyx cython-0.20.1+1~202203241016-9537/tests/run/cstringmul.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cstringmul.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cstringmul.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,9 +1,45 @@
 __doc__ = u"""
-    >>> print(spam)
-    eggseggseggseggs
-    >>> print(grail)
-    tomatotomatotomatotomatotomatotomatotomato
+>>> print(spam)
+eggseggseggseggs
+>>> print(uspam)
+eggseggseggseggs
+>>> print(bspam.decode('ascii'))
+eggseggseggseggs
+
+>>> print(grail)
+tomatotomatotomatotomatotomatotomatotomato
+>>> len(grail_long)
+4200
+>>> print(ugrail)
+tomatotomatotomatotomatotomatotomatotomato
+>>> len(ugrail_long)
+4200
+>>> print(bgrail.decode('ascii'))
+tomatotomatotomatotomatotomatotomatotomato
+>>> len(bgrail_long)
+4200
 """
 
-spam = u"eggs" * 4
-grail = 7 * u"tomato"
+bspam = b"eggs" * 4
+bgrail = 7 * b"tomato"
+bgrail_long = 700 * b"tomato"
+
+spam = "eggs" * 4
+grail = 7 * "tomato"
+grail_long = 700 * "tomato"
+
+uspam = u"eggs" * 4
+ugrail = 7 * u"tomato"
+ugrail_long = 700 * u"tomato"
+
+cimport cython
+
+@cython.test_assert_path_exists("//StringNode[@value = '-----']")
+@cython.test_assert_path_exists("//StringNode[@unicode_value = '-----']")
+def gh3951():
+    """
+    Bug occurs with language_level=2 and affects StringNode.value
+    >>> gh3951()
+    '-----'
+    """
+    return "-"*5
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cstruct.pyx cython-0.20.1+1~202203241016-9537/tests/run/cstruct.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cstruct.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cstruct.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -62,3 +62,22 @@
     cdef Grail l
     spam.g = &l
     eggs_g(spam)
+
+
+cdef struct Ints:
+    int a, b
+
+def assign_fields_in_loop():
+    """
+    >>> assign_fields_in_loop()
+    2
+    """
+    cdef int i = 0
+    cdef Ints s
+    for s.a, s.b in enumerate(range(3)):
+        assert s.a == s.b
+        assert s.a == i
+        i += 1
+
+    assert s.a == s.b
+    return s.b
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ct_DEF.pyx cython-0.20.1+1~202203241016-9537/tests/run/ct_DEF.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ct_DEF.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ct_DEF.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,5 @@
+# mode: run
+# tag: warnings
 
 cimport cython
 
@@ -232,3 +234,30 @@
     >>> none()
     """
     return NONE
+
+
+_WARNINGS = """
+24:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+25:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+26:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+28:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+29:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+30:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+31:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+32:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+33:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+34:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+35:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+36:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+37:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+38:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+39:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+40:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+41:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+42:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+43:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+44:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+45:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+46:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+47:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ct_IF.pyx cython-0.20.1+1~202203241016-9537/tests/run/ct_IF.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ct_IF.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ct_IF.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,6 @@
+# mode: run
+# tag: warnings
+
 DEF NO = 0
 DEF YES = 1
 
@@ -42,3 +45,49 @@
     ELSE:
         i = 3
     return i
+
+
+def control_flow_DEF1():
+    """
+    >>> control_flow_DEF1()
+    B should be 2.
+    2
+    """
+    IF YES:
+        DEF B=2
+        print('B should be 2.')
+    ELSE:
+        DEF B=3
+        print('B should be 3.')
+    return B
+
+
+def control_flow_DEF2():
+    """
+    >>> control_flow_DEF2()
+    B should be 3.
+    3
+    """
+    IF NO:
+        DEF B=2
+        print('B should be 2.')
+    ELSE:
+        DEF B=3
+        print('B should be 3.')
+    return B
+
+
+_WARNINGS = """
+13:4: The 'IF' statement is deprecated and will be removed in a future Cython version. Consider using runtime conditions or C macros instead. See https://github.com/cython/cython/issues/4310
+27:4: The 'IF' statement is deprecated and will be removed in a future Cython version. Consider using runtime conditions or C macros instead. See https://github.com/cython/cython/issues/4310
+41:4: The 'IF' statement is deprecated and will be removed in a future Cython version. Consider using runtime conditions or C macros instead. See https://github.com/cython/cython/issues/4310
+56:4: The 'IF' statement is deprecated and will be removed in a future Cython version. Consider using runtime conditions or C macros instead. See https://github.com/cython/cython/issues/4310
+71:4: The 'IF' statement is deprecated and will be removed in a future Cython version. Consider using runtime conditions or C macros instead. See https://github.com/cython/cython/issues/4310
+
+4:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+5:0: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+57:8: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+60:8: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+72:8: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+75:8: The 'DEF' statement is deprecated and will be removed in a future Cython version. Consider using global variables, constants, and in-place literals instead. See https://github.com/cython/cython/issues/4310
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ctuple.pyx cython-0.20.1+1~202203241016-9537/tests/run/ctuple.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ctuple.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ctuple.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -142,6 +142,17 @@
     return x, y
 
 
+def cast_to_ctuple(*o):
+    """
+    >>> cast_to_ctuple(1, 2.)
+    (1, 2.0)
+    """
+    cdef int x
+    cdef double y
+    x, y = <(int, double)>o
+    return x, y
+
+
 @cython.infer_types(True)
 def test_type_inference():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ctypedef_bint.pyx cython-0.20.1+1~202203241016-9537/tests/run/ctypedef_bint.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ctypedef_bint.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ctypedef_bint.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,71 @@
+from __future__ import print_function
+
+from cython cimport typeof
+
+ctypedef bint mybool
+
+cdef mybool mybul = True
+cdef bint bul = True
+cdef int num = 42
+
+
+def CondExprNode_to_obj(test):
+    """
+    >>> CondExprNode_to_obj(True)
+    Python object | Python object
+    2
+    >>> CondExprNode_to_obj(False)
+    Python object | Python object
+    84
+    """
+
+    print(typeof(mybul if test else num), "|", typeof(bul if test else num))
+
+    return (mybul if test else num) + (bul if test else num)
+
+
+def BoolBinopNode_to_obj():
+    """
+    >>> BoolBinopNode_to_obj()
+    Python object | Python object
+    2
+    """
+
+    print(typeof(mybul or num), "|", typeof(bul or num))
+
+    return (mybul or num) + (bul or num)
+
+
+cdef int test_bool(mybool arg):
+    return arg
+
+
+def CondExprNode_to_bool(test):
+    """
+    >>> CondExprNode_to_bool(True)
+    bint | bint
+    0
+    >>> CondExprNode_to_bool(False)
+    bint | bint
+    2
+    """
+
+    print(typeof(not mybul if test else mybul), "|", typeof(not bul if test else bul))
+
+    # test_bool() would silently crash if one of the types is cast
+    # to Python object and not just assigned.
+    # It happens when a type is wrongly inferred as Python object
+    # instead of bint or mybool.
+    return test_bool(not mybul if test else mybul) + test_bool(not bul if test else bul)
+
+
+def BoolBinopNode_to_bool():
+    """
+    >>> BoolBinopNode_to_bool()
+    bint | bint
+    2
+    """
+
+    print(typeof(not mybul or mybul), "|", typeof(not bul or bul))
+
+    return test_bool(not mybul or mybul) + test_bool(not bul or bul)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ctypedef_int_types_T333.pyx cython-0.20.1+1~202203241016-9537/tests/run/ctypedef_int_types_T333.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ctypedef_int_types_T333.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ctypedef_int_types_T333.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 333
+# ticket: t333
 #cython: autotestdict=True
 
 # -------------------------------------------------------------------
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/c_type_methods_T236.pyx cython-0.20.1+1~202203241016-9537/tests/run/c_type_methods_T236.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/c_type_methods_T236.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/c_type_methods_T236.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,10 +1,8 @@
-# ticket: 236
-
-__doc__ = ''
+# ticket: t236
 
 import sys
-if sys.version_info >= (2,6):
-    __doc__ += '''
+
+__doc__ = '''
 >>> float_is_integer(1.0)
 True
 >>> float_is_integer(1.1)
@@ -19,7 +17,6 @@
 '''
 
 def float_is_integer(float f):
-    # requires Python 2.6+
     return f.is_integer()
 
 def int_bit_length(int i):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cyfunction_defaults.pyx cython-0.20.1+1~202203241016-9537/tests/run/cyfunction_defaults.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cyfunction_defaults.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cyfunction_defaults.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -86,10 +86,49 @@
         return a
     return func
 
+def assign_defaults_and_check_warnings(func, value=None, delete=False):
+    import warnings
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        if delete:
+            del func.__defaults__
+        else:
+            func.__defaults__ = value
+        assert len(w) == 1, len(w)
+        assert issubclass(w[0].category, RuntimeWarning), w[0].category
+        assert "changes to cyfunction.__defaults__" in str(w[0].message), str(w[0].message)
+
+def test_assign_defaults():
+    """
+    >>> f = test_assign_defaults()
+    >>> f.__defaults__
+    (5, 10)
+    >>> assign_defaults_and_check_warnings(f, value=())
+    >>> f.__defaults__
+    ()
+    >>> assign_defaults_and_check_warnings(f, delete=True)
+    >>> f.__defaults__
+    >>> f.__defaults__ = "Not a tuple"
+    Traceback (most recent call last):
+    TypeError: __defaults__ must be set to a tuple object
+    """
+    def func(a=5, b=10):
+        return a, b
+    return func
+
 
 def cy_kwonly_default_args(a, x=1, *, b=2):
     l = m = 1
 
+def assign_kwdefaults_and_check_warnings(func, value):
+    import warnings
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("always")
+        func.__kwdefaults__ = value
+        assert len(w) == 1, len(w)
+        assert issubclass(w[0].category, RuntimeWarning), w[0].category
+        assert "changes to cyfunction.__kwdefaults__" in str(w[0].message), str(w[0].message)
+
 def test_kwdefaults(value):
     """
     >>> cy_kwonly_default_args.__defaults__
@@ -111,12 +150,12 @@
     >>> f.__kwdefaults__ = ()
     Traceback (most recent call last):
     TypeError: __kwdefaults__ must be set to a dict object
-    >>> f.__kwdefaults__ = None
+    >>> assign_kwdefaults_and_check_warnings(f, None)
     >>> f.__kwdefaults__
-    >>> f.__kwdefaults__ = {}
+    >>> assign_kwdefaults_and_check_warnings(f, {})
     >>> f.__kwdefaults__
     {}
-    >>> f.__kwdefaults__ = {'a': 2}
+    >>> assign_kwdefaults_and_check_warnings(f, {'a': 2})
     >>> f.__kwdefaults__
     {'a': 2}
     """
@@ -171,6 +210,31 @@
         print "i", i, "func result", f(1.0), "defaults", get_defaults(f)
 
 
+def test_memoryview_none(const unsigned char[:] b=None):
+    """
+    >>> test_memoryview_none()
+    >>> test_memoryview_none(None)
+    >>> test_memoryview_none(b'abc')
+    97
+    """
+    if b is None:
+        return None
+    return b[0]
+
+
+def test_memoryview_bytes(const unsigned char[:] b=b'xyz'):
+    """
+    >>> test_memoryview_bytes()
+    120
+    >>> test_memoryview_bytes(None)
+    >>> test_memoryview_bytes(b'abc')
+    97
+    """
+    if b is None:
+        return None
+    return b[0]
+
+
 @cython.test_fail_if_path_exists(
     '//NameNode[@entry.in_closure = True]',
     '//NameNode[@entry.from_closure = True]')
@@ -226,3 +290,62 @@
         return arg
     print i  # genexprs don't leak
     return func
+
+cdef class C:
+    def f1(self, a, b=1, c=[]):
+        pass
+    def f2(self, a, b=1,/, c=[1]):
+        pass
+    def f3(self, a, /, b=1, *, c=[1]):
+        pass
+    cpdef f4(self, a, char*c=NULL):
+        pass
+    cpdef f5(self, a, str s = "123"):
+        pass
+    cpdef f6(self, a, int s = 4):
+        pass
+    cpdef f7(self, a, dict s = {'a':22}):
+        pass
+    cpdef f8(self, a, list s = [15]):
+        pass
+    cpdef f9(self, a, int[:] s = None):
+        pass
+    def f10(self, a, /, b=1, *, int[:] c=None):
+        pass
+
+
+def check_defaults_on_methods_for_introspection():
+    """
+    >>> C.f1.__defaults__
+    (1, [])
+    >>> C.f1.__kwdefaults__
+    >>> C.f2.__defaults__
+    (1, [1])
+    >>> C.f2.__kwdefaults__
+    >>> C.f3.__defaults__
+    (1,)
+    >>> C.f3.__kwdefaults__
+    {'c': [1]}
+    >>> C.f4.__defaults__
+    >>> C.f4.__kwdefaults__
+    >>> C.f5.__defaults__
+    ('123',)
+    >>> C.f5.__kwdefaults__
+    >>> C.f6.__defaults__
+    (4,)
+    >>> C.f6.__kwdefaults__
+    >>> C.f7.__defaults__
+    ({'a': 22},)
+    >>> C.f7.__kwdefaults__
+    >>> C.f8.__defaults__
+    ([15],)
+    >>> C.f8.__kwdefaults__
+    >>> C.f9.__defaults__
+    (None,)
+    >>> C.f9.__kwdefaults__
+    >>> C.f10.__defaults__
+    (1,)
+    >>> C.f10.__kwdefaults__
+    {'c': None}
+    """
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cyfunction_METH_O_GH1728.pyx cython-0.20.1+1~202203241016-9537/tests/run/cyfunction_METH_O_GH1728.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cyfunction_METH_O_GH1728.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cyfunction_METH_O_GH1728.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,16 @@
+# cython: binding=True
+# mode: run
+# tag: cyfunction
+
+cdef class TestMethodOneArg:
+    def meth(self, arg):
+        pass
+
+def call_meth(x):
+    """
+    >>> call_meth(TestMethodOneArg())  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: meth() takes exactly ... argument (0 given)
+    """
+    return x.meth()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cyfunction.pyx cython-0.20.1+1~202203241016-9537/tests/run/cyfunction.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cyfunction.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cyfunction.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,6 +3,7 @@
 # tag: cyfunction
 
 import sys
+IS_PY2 = sys.version_info[0] < 3
 IS_PY3 = sys.version_info[0] >= 3
 IS_PY34 = sys.version_info > (3, 4, 0, 'beta', 3)
 
@@ -270,13 +271,13 @@
     >>> isinstance(test_annotations.__annotations__, dict)
     True
     >>> sorted(test_annotations.__annotations__.items())
-    [('a', 'test'), ('b', 'other'), ('c', 123), ('return', 'ret')]
+    [('a', "'test'"), ('b', "'other'"), ('c', '123'), ('return', "'ret'")]
 
     >>> def func_b(): return 42
     >>> def func_c(): return 99
     >>> inner = test_annotations(1, func_b, func_c)
     >>> sorted(inner.__annotations__.items())
-    [('return', 99), ('x', 'banana'), ('y', 42)]
+    [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')]
 
     >>> inner.__annotations__ = {234: 567}
     >>> inner.__annotations__
@@ -292,15 +293,132 @@
 
     >>> inner = test_annotations(1, func_b, func_c)
     >>> sorted(inner.__annotations__.items())
-    [('return', 99), ('x', 'banana'), ('y', 42)]
+    [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')]
     >>> inner.__annotations__['abc'] = 66
     >>> sorted(inner.__annotations__.items())
-    [('abc', 66), ('return', 99), ('x', 'banana'), ('y', 42)]
+    [('abc', 66), ('return', 'c()'), ('x', "'banana'"), ('y', 'b()')]
 
     >>> inner = test_annotations(1, func_b, func_c)
     >>> sorted(inner.__annotations__.items())
-    [('return', 99), ('x', 'banana'), ('y', 42)]
+    [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')]
     """
     def inner(x: "banana", y: b()) -> c():
         return x,y
     return inner
+
+
+def add_one(func):
+    "Decorator to add 1 to the last argument of the function call"
+    def inner(*args):
+        args = args[:-1] + (args[-1] + 1,)
+        return func(*args)
+    return inner
+
+@add_one
+def test_decorated(x):
+    """
+    >>> test_decorated(0)
+    1
+    """
+    return x
+
+@add_one
+@add_one
+def test_decorated2(x):
+    """
+    >>> test_decorated2(0)
+    2
+    """
+    return x
+
+
+cdef class TestDecoratedMethods:
+    @add_one
+    def test(self, x):
+        """
+        >>> TestDecoratedMethods().test(0)
+        1
+        """
+        return x
+
+    @add_one
+    @add_one
+    def test2(self, x):
+        """
+        >>> TestDecoratedMethods().test2(0)
+        2
+        """
+        return x
+
+    def test_calls(self, x):
+        """
+        >>> TestDecoratedMethods().test_calls(2)
+        25
+        """
+        return self.test(x) + self.test2(x*10)
+
+
+cdef class TestUnboundMethodCdef:
+    """
+    >>> C = TestUnboundMethodCdef
+    >>> IS_PY2 or (C.meth is C.__dict__["meth"])
+    True
+    """
+    def meth(self): pass
+
+
+class TestUnboundMethod:
+    """
+    >>> C = TestUnboundMethod
+    >>> IS_PY2 or (C.meth is C.__dict__["meth"])
+    True
+    """
+    def meth(self): pass
+
+
+class TestStaticmethod(object):
+    """
+    >>> x = TestStaticmethod()
+    >>> x.staticmeth(42)
+    42
+    >>> x.staticmeth.__get__(42)()
+    42
+    """
+    @staticmethod
+    def staticmeth(arg): return arg
+
+
+cdef class TestOptimisedBuiltinMethod:
+    """
+    >>> obj = TestOptimisedBuiltinMethod()
+    >>> obj.append(2)
+    3
+    >>> obj.call(2)
+    4
+    >>> obj.call(3, obj)
+    5
+    """
+    def append(self, arg):
+        print(arg+1)
+
+    def call(self, arg, obj=None):
+        (obj or self).append(arg+1)  # optimistically optimised => uses fast fallback method call
+
+
+def do_nothing(f):
+    """Dummy decorator for `test_firstlineno_decorated_function`"""
+    return f
+
+
+@do_nothing
+@do_nothing
+def test_firstlineno_decorated_function():
+    """
+    check that `test_firstlineno_decorated_function` starts 5 lines below `do_nothing`
+
+    >>> test_firstlineno_decorated_function()
+    5
+    """
+    l1 = do_nothing.__code__.co_firstlineno
+    l2 = test_firstlineno_decorated_function.__code__.co_firstlineno
+    return l2 - l1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cython3_no_unicode_literals.pyx cython-0.20.1+1~202203241016-9537/tests/run/cython3_no_unicode_literals.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cython3_no_unicode_literals.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cython3_no_unicode_literals.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,172 @@
+# cython: language_level=3str, binding=True
+# mode: run
+# tag: python3, str_is_str
+
+print(end='')  # test that language_level 3 applies immediately at the module start, for the first token.
+
+__doc__ = """
+>>> items = sorted(locals_function(1).items())
+>>> for item in items:
+...     print('%s = %r' % item)
+a = 1
+b = 2
+x = 'abc'
+"""
+
+import sys
+IS_PY2 = sys.version_info[0] < 3
+
+
+def locals_function(a, b=2):
+    x = 'abc'
+    return locals()
+
+
+### true division
+
+def truediv(x):
+    """
+    >>> truediv(4)
+    2.0
+    >>> truediv(3)
+    1.5
+    """
+    return x / 2
+
+
+def truediv_int(int x):
+    """
+    >>> truediv_int(4)
+    2.0
+    >>> truediv_int(3)
+    1.5
+    """
+    return x / 2
+
+
+### Py3 feature tests
+
+def print_function(*args):
+    """
+    >>> print_function(1,2,3)
+    1 2 3
+    """
+    print(*args) # this isn't valid Py2 syntax
+
+
+str_string = "abcdefg"
+
+def no_unicode_literals():
+    """
+    >>> print( no_unicode_literals() )
+    True
+    abcdefg
+
+    Testing non-ASCII docstrings: Πυθαγόρας
+    """
+    print(isinstance(str_string, str) or type(str_string))
+    return str_string
+
+
+def non_ascii_str():
+    u"""
+    >>> s = 'ø\\x20\\u0020'
+    >>> isinstance(s, str)
+    True
+    >>> print(not IS_PY2 or len(s) == 9 or len(s))  # first is 2-char bytes in Py2, hex escape is resolved
+    True
+    >>> print(IS_PY2 or len(s) == 3 or len(s))      # 3 unicode characters in Py3
+    True
+
+    >>> s = non_ascii_str()
+    >>> isinstance(s, str)
+    True
+    >>> print(not IS_PY2 or len(s) == 9 or len(s))  # first is 2-char bytes in Py2, hex escape is resolved
+    True
+    >>> print(IS_PY2 or len(s) == 3 or len(s))      # 3 unicode characters in Py3
+    True
+    """
+    s = 'ø\x20\u0020'
+    assert isinstance(s, str)
+    assert (IS_PY2 and isinstance(s, bytes)) or (not IS_PY2 and isinstance(s, unicode))
+    return s
+
+
+def non_ascii_raw_str():
+    u"""
+    >>> s = r'ø\\x20\\u0020'
+    >>> print(not IS_PY2 or len(s) == 12 or len(s))  # Py2 (first character is two bytes)
+    True
+    >>> print(IS_PY2 or len(s) == 11 or len(s))      # Py3 (unicode string)
+    True
+
+    >>> s = non_ascii_raw_str()
+    >>> isinstance(s, str)
+    True
+    >>> print(not IS_PY2 or len(s) == 12 or len(s))  # Py2 (first character is two bytes)
+    True
+    >>> print(IS_PY2 or len(s) == 11 or len(s))      # Py3 (unicode string)
+    True
+    """
+    s = r'ø\x20\u0020'
+    assert isinstance(s, str)
+    assert (IS_PY2 and isinstance(s, bytes)) or (not IS_PY2 and isinstance(s, unicode))
+    return s
+
+
+def non_ascii_raw_unicode():
+    u"""
+    >>> s = non_ascii_raw_unicode()
+    >>> isinstance(s, bytes)
+    False
+    >>> len(s)
+    11
+    """
+    s = ru'ø\x20\u0020'
+    assert isinstance(s, unicode)
+    return s
+
+
+def str_type_is_str():
+    """
+    >>> str_type, s = str_type_is_str()
+    >>> isinstance(s, type(str_string)) or (s, str_type)
+    True
+    >>> isinstance(s, str_type) or (s, str_type)
+    True
+    >>> isinstance(str_string, str_type) or str_type
+    True
+    """
+    cdef str s = 'abc'
+    return str, s
+
+def strip_wrapped_string(s):
+    # PEP 563 translates an annotation of "test new test" to '"test new test"'
+    # but choice of string delimiters is a bit arbitrary
+    #  this function handles that
+    assert s[0] == s[-1] # delimiters on either end are the same
+    return s[1:-1] # strip them
+
+def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwargs: "KWARGS") -> "ret":
+    """
+    >>> annotation_syntax(1)
+    3
+    >>> annotation_syntax(1,3)
+    4
+
+    >>> len(annotation_syntax.__annotations__)
+    5
+    >>> strip_wrapped_string(annotation_syntax.__annotations__['a'])
+    'test new test'
+    >>> strip_wrapped_string(annotation_syntax.__annotations__['b'])
+    'other'
+    >>> strip_wrapped_string(annotation_syntax.__annotations__['args'])
+    'ARGS'
+    >>> strip_wrapped_string(annotation_syntax.__annotations__['kwargs'])
+    'KWARGS'
+    >>> strip_wrapped_string(annotation_syntax.__annotations__['return'])
+    'ret'
+    """
+    result : int = a + b
+
+    return result
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cython3.pyx cython-0.20.1+1~202203241016-9537/tests/run/cython3.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cython3.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cython3.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,8 @@
-# cython: language_level=3, binding=True
+# cython: language_level=3, binding=True, annotation_typing=False
 # mode: run
-# tag: generators, python3, exceptions
+# tag: generators, python3, exceptions, gh2230, gh2811
+
+print(end='')  # test that language_level 3 applies immediately at the module start, for the first token.
 
 cimport cython
 
@@ -14,19 +16,34 @@
 
 >>> except_as_deletes
 True
+
 >>> no_match_does_not_touch_target
 True
 """
 
 import sys
-if sys.version_info[0] >= 3:
+IS_PY2 = sys.version_info[0] < 3
+if not IS_PY2:
     __doc__ = __doc__.replace(" u'", " '")
 
+
 def locals_function(a, b=2):
     x = 'abc'
     return locals()
 
 
+### "new style" classes
+
+class T:
+    """
+    >>> t = T()
+    >>> isinstance(t, T)
+    True
+    >>> isinstance(T, type)  # not a Py2 old style class!
+    True
+    """
+
+
 ### true division
 
 def truediv(x):
@@ -255,6 +272,32 @@
         yield 6
 
 
+def nested_except_gh3666(a=False, b=False):
+    """
+    >>> print(nested_except_gh3666())
+    A
+    >>> print(nested_except_gh3666(a=True))
+    B-V
+    >>> print(nested_except_gh3666(a=True, b=True))
+    B-V-T
+    """
+    try:
+        if a:
+            raise ValueError
+        return "A"
+    except TypeError as exc:
+        return "A-T"
+    except ValueError as exc:
+        try:
+            if b:
+                raise TypeError
+            return "B-V"
+        except ValueError as exc:
+            return "B-V-V"
+        except TypeError as exc:
+            return "B-V-T"
+
+
 ### Py3 feature tests
 
 def print_function(*args):
@@ -275,6 +318,7 @@
     exec(cmd, g, l)
     return l
 
+
 def exec2_function(cmd):
     """
     >>> exec2_function('a = 1+1')['a']
@@ -284,6 +328,7 @@
     exec(cmd, g)
     return g
 
+
 EXEC_GLOBAL = [5]
 
 def exec1_function(cmd):
@@ -295,6 +340,7 @@
     exec(cmd)
     return EXEC_GLOBAL[old:]
 
+
 ustring = "abcdefg"
 
 def unicode_literals():
@@ -306,6 +352,46 @@
     print(isinstance(ustring, unicode) or type(ustring))
     return ustring
 
+
+def non_ascii_unprefixed_str():
+    """
+    >>> s = non_ascii_unprefixed_str()
+    >>> isinstance(s, bytes)
+    False
+    >>> len(s)
+    3
+    """
+    s = 'ø\x20\u0020'
+    assert isinstance(s, unicode)
+    return s
+
+
+def non_ascii_raw_str():
+    """
+    >>> s = non_ascii_raw_str()
+    >>> isinstance(s, bytes)
+    False
+    >>> len(s)
+    11
+    """
+    s = r'ø\x20\u0020'
+    assert isinstance(s, unicode)
+    return s
+
+
+def non_ascii_raw_prefixed_unicode():
+    """
+    >>> s = non_ascii_raw_prefixed_unicode()
+    >>> isinstance(s, bytes)
+    False
+    >>> len(s)
+    11
+    """
+    s = ru'ø\x20\u0020'
+    assert isinstance(s, unicode)
+    return s
+
+
 def str_type_is_unicode():
     """
     >>> str_type, s = str_type_is_unicode()
@@ -319,6 +405,7 @@
     cdef str s = 'abc'
     return str, s
 
+
 def loop_over_unicode_literal():
     """
     >>> print( loop_over_unicode_literal() )
@@ -329,6 +416,7 @@
         assert uchar in 'abcdefg'
     return cython.typeof(uchar)
 
+
 def list_comp():
     """
     >>> list_comp()
@@ -339,6 +427,28 @@
     assert x == 'abc' # don't leak in Py3 code
     return result
 
+
+def list_comp_iterable(it):
+    """
+    >>> list_comp_iterable([])
+    []
+    >>> list_comp_iterable([0])
+    [0]
+    >>> list_comp_iterable([1])
+    []
+    >>> list_comp_iterable([0, 1])
+    [0]
+    >>> list_comp_iterable([2])
+    [4]
+    >>> list_comp_iterable(range(5))
+    [0, 4, 8]
+    """
+    x = 'abc'
+    result = [x*2 for x in it if x % 2 == 0]
+    assert x == 'abc' # don't leak in Py3 code
+    return result
+
+
 def list_comp_with_lambda():
     """
     >>> list_comp_with_lambda()
@@ -349,6 +459,25 @@
     assert x == 'abc' # don't leak in Py3 code
     return result
 
+
+class ListCompInClass(object):
+    """
+    >>> x = ListCompInClass()
+    >>> x.listcomp
+    [1, 2, 3]
+    """
+    listcomp = [i+1 for i in range(3)]
+
+
+cdef class ListCompInCClass:
+    """
+    >>> x = ListCompInCClass()
+    >>> x.listcomp
+    [1, 2, 3]
+    """
+    listcomp = [i+1 for i in range(3)]
+
+
 module_level_lc = [ module_level_loopvar*2 for module_level_loopvar in range(4) ]
 def list_comp_module_level():
     """
@@ -359,6 +488,7 @@
     NameError: ...name 'module_level_loopvar' is not defined
     """
 
+
 module_level_list_genexp = list(module_level_genexp_loopvar*2 for module_level_genexp_loopvar in range(4))
 def genexpr_module_level():
     """
@@ -369,6 +499,7 @@
     NameError: ...name 'module_level_genexp_loopvar' is not defined
     """
 
+
 def list_comp_unknown_type(l):
     """
     >>> list_comp_unknown_type(range(5))
@@ -376,6 +507,7 @@
     """
     return [x*2 for x in l if x % 2 == 0]
 
+
 def listcomp_as_condition(sequence):
     """
     >>> listcomp_as_condition(['a', 'b', '+'])
@@ -389,6 +521,7 @@
         return True
     return False
 
+
 def set_comp():
     """
     >>> sorted(set_comp())
@@ -399,6 +532,7 @@
     assert x == 'abc' # don't leak
     return result
 
+
 def dict_comp():
     """
     >>> sorted(dict_comp().items())
@@ -409,6 +543,7 @@
     assert x == 'abc' # don't leak
     return result
 
+
 # in Python 3, d.keys/values/items() are the iteration methods
 @cython.test_assert_path_exists(
     "//WhileStatNode",
@@ -434,6 +569,7 @@
     items = [ item for item in d.items() ]
     return keys, values, items
 
+
 @cython.test_assert_path_exists(
     "//WhileStatNode",
     "//WhileStatNode//DictIterationNextNode")
@@ -457,6 +593,7 @@
     items = [ item for item in {11 : 1, 22 : 2, 33 : 3}.items() ]
     return dict_keys, keys, values, items
 
+
 def int_literals():
     """
     >>> int_literals()
@@ -470,6 +607,7 @@
     print(cython.typeof(1UL))
     print(cython.typeof(10000000000000UL))
 
+
 def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwargs: "KWARGS") -> "ret":
     """
     >>> annotation_syntax(1)
@@ -480,14 +618,49 @@
     >>> len(annotation_syntax.__annotations__)
     5
     >>> print(annotation_syntax.__annotations__['a'])
-    test new test
+    'test new test'
     >>> print(annotation_syntax.__annotations__['b'])
-    other
+    'other'
     >>> print(annotation_syntax.__annotations__['args'])
-    ARGS
+    'ARGS'
     >>> print(annotation_syntax.__annotations__['kwargs'])
-    KWARGS
+    'KWARGS'
     >>> print(annotation_syntax.__annotations__['return'])
-    ret
+    'ret'
+    """
+    result : int = a + b
+
+    return result
+
+
+def builtin_as_annotation(text: str):
+    # See https://github.com/cython/cython/issues/2811
+    """
+    >>> builtin_as_annotation("abc")
+    a
+    b
+    c
+    """
+    for c in text:
+        print(c)
+
+
+async def async_def_annotations(x: 'int') -> 'float':
+    """
+    >>> ret, arg = sorted(async_def_annotations.__annotations__.items())
+    >>> print(ret[0]); print(ret[1])
+    return
+    'float'
+    >>> print(arg[0]); print(arg[1])
+    x
+    'int'
+    """
+    return float(x)
+
+
+def repr_returns_str(x) -> str:
+    """
+    >>> repr_returns_str(123)
+    '123'
     """
-    return a+b
+    return repr(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/cython_includes.pyx cython-0.20.1+1~202203241016-9537/tests/run/cython_includes.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/cython_includes.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/cython_includes.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -4,6 +4,59 @@
 from cpython cimport PyType_Check as PyType_Check2
 from cpython.type cimport PyType_Check as PyType_Check3
 
+# Make sure we can cimport all .pxd files.
+cimport cpython.array
+cimport cpython.bool
+cimport cpython.buffer
+cimport cpython.bytearray
+cimport cpython.bytes
+cimport cpython.cellobject
+cimport cpython.ceval
+cimport cpython.cobject
+cimport cpython.codecs
+cimport cpython.complex
+cimport cpython.contextvars
+cimport cpython.conversion
+cimport cpython.datetime
+cimport cpython.dict
+cimport cpython.exc
+cimport cpython.fileobject
+cimport cpython.float
+cimport cpython.function
+cimport cpython.genobject
+cimport cpython.getargs
+cimport cpython.instance
+cimport cpython.int
+cimport cpython.iterator
+cimport cpython.iterobject
+cimport cpython.list
+cimport cpython.long
+cimport cpython.longintrepr
+cimport cpython.mapping
+cimport cpython.marshal
+cimport cpython.mem
+cimport cpython.memoryview
+cimport cpython.method
+cimport cpython.module
+cimport cpython.number
+cimport cpython.object
+cimport cpython.oldbuffer
+cimport cpython.pycapsule
+cimport cpython.pylifecycle
+cimport cpython.pystate
+cimport cpython.pythread
+cimport cpython.ref
+cimport cpython.sequence
+cimport cpython.set
+cimport cpython.slice
+cimport cpython.string
+cimport cpython.tuple
+cimport cpython.type
+cimport cpython.unicode
+cimport cpython.version
+cimport cpython.weakref
+
+
 def libc_cimports():
     """
     >>> libc_cimports()
@@ -13,6 +66,7 @@
     sprintf(buf, "%s", b'hello')
     print (buf).decode('ASCII')
 
+
 def cpython_cimports():
     """
     >>> cpython_cimports()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/datetime_cimport.pyx cython-0.20.1+1~202203241016-9537/tests/run/datetime_cimport.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/datetime_cimport.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/datetime_cimport.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,10 +1,12 @@
 # coding: utf-8
 
 from cpython.datetime cimport import_datetime
-from cpython.datetime cimport date, time, datetime, timedelta, PyDateTime_IMPORT
+from cpython.datetime cimport date, time, datetime, timedelta, timezone_new, PyDateTime_IMPORT
+
+import sys
 
 import_datetime()
-        
+
 def test_date(int year, int month, int day):
     '''
     >>> val = test_date(2012, 12, 31)
@@ -40,3 +42,20 @@
     '''
     val = timedelta(days, seconds, useconds)
     return val
+
+def test_timezone(int days, int seconds, int useconds, str name):
+    '''
+    >>> val = test_timezone(0, 3600, 0, 'CET')
+    >>> print(val)
+    True
+    '''
+    try:
+        val = timezone_new(timedelta(days, seconds, useconds), name)
+    except RuntimeError:
+        if sys.version_info < (3, 7):
+            return True
+        else:
+            # It's only supposed to raise on Python < 3.7
+            return False
+    else:
+        return True
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/datetime_members.pyx cython-0.20.1+1~202203241016-9537/tests/run/datetime_members.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/datetime_members.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/datetime_members.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,11 +1,17 @@
+# mode: run
+# tag: datetime
+
+import sys
+
 from cpython.datetime cimport import_datetime
 from cpython.datetime cimport time_new, date_new, datetime_new, timedelta_new
+from cpython.datetime cimport datetime, time
 from cpython.datetime cimport time_tzinfo, datetime_tzinfo
-from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond
+from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond, time_tzinfo, time_fold
 from cpython.datetime cimport date_day, date_month, date_year
 from cpython.datetime cimport datetime_day, datetime_month, datetime_year
 from cpython.datetime cimport datetime_hour, datetime_minute, datetime_second, \
-                              datetime_microsecond
+                              datetime_microsecond, datetime_tzinfo, datetime_fold
 from cpython.datetime cimport timedelta_days, timedelta_seconds, timedelta_microseconds
 
 import_datetime()
@@ -20,31 +26,41 @@
            o.month == date_month(o), \
            o.day == date_day(o)
 
-def test_datetime(int year, int month, int day, 
-                  int hour, int minute, int second, int microsecond):
-    '''
-    >>> test_datetime(2012, 12, 31, 12, 30, 59, 12345)
-    (True, True, True, True, True, True, True)
+def test_datetime(int year, int month, int day, int hour,
+                  int minute, int second, int microsecond, int fold):
     '''
-    o = datetime_new(year, month, day, hour, minute, second, microsecond, None)
+    >>> test_datetime(2012, 12, 31, 12, 30, 59, 12345, 0)
+    (True, True, True, True, True, True, True, True, True)
+    >>> test_datetime(2012, 12, 11, 12, 30, 59, 3322, 1 if sys.version_info >= (3, 7) else 0)
+    (True, True, True, True, True, True, True, True, True)
+    '''
+    o = datetime_new(
+        year, month, day, hour, minute, second, microsecond, None, fold
+    )
     return o.year == datetime_year(o), \
            o.month == datetime_month(o), \
            o.day == datetime_day(o), \
            o.hour == datetime_hour(o), \
            o.minute == datetime_minute(o), \
            o.second == datetime_second(o), \
-           o.microsecond == datetime_microsecond(o)
-
-def test_time(int hour, int minute, int second, int microsecond):
-    '''
-    >>> test_time(12, 30, 59, 12345)
-    (True, True, True, True)
+           o.microsecond == datetime_microsecond(o), \
+           o.tzinfo == datetime_tzinfo(o), \
+           o.fold == datetime_fold(o)
+
+def test_time(int hour, int minute, int second, int microsecond, int fold):
+    '''
+    >>> test_time(12, 30, 59, 12345, 0)
+    (True, True, True, True, True, True)
+    >>> test_time(12, 30, 43, 5432, 1 if sys.version_info >= (3, 7) else 0)
+    (True, True, True, True, True, True)
     '''
-    o = time_new(hour, minute, second, microsecond, None)
+    o = time_new(hour, minute, second, microsecond, None, fold)
     return o.hour == time_hour(o), \
            o.minute == time_minute(o), \
            o.second == time_second(o), \
-           o.microsecond == time_microsecond(o)
+           o.microsecond == time_microsecond(o), \
+           o.tzinfo == time_tzinfo(o), \
+           o.fold == time_fold(o)
 
 def test_timedelta(int days, int seconds, int microseconds):
     '''
@@ -55,4 +71,3 @@
     return o.days == timedelta_days(o), \
            o.seconds == timedelta_seconds(o), \
            o.microseconds == timedelta_microseconds(o)
-
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/datetime_pxd.pyx cython-0.20.1+1~202203241016-9537/tests/run/datetime_pxd.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/datetime_pxd.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/datetime_pxd.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,10 +1,8 @@
 # coding: utf-8
 
-#cimport cpython.datetime as cy_datetime
-#from datetime import time, date, datetime, timedelta, tzinfo
+cimport cython
 
-
-from cpython.datetime cimport import_datetime
+from cpython.datetime cimport import_datetime, timedelta
 from cpython.datetime cimport time_new, date_new, datetime_new, timedelta_new
 from cpython.datetime cimport time_tzinfo, datetime_tzinfo
 from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond
@@ -12,8 +10,20 @@
 from cpython.datetime cimport datetime_day, datetime_month, datetime_year
 from cpython.datetime cimport datetime_hour, datetime_minute, datetime_second, \
                               datetime_microsecond
+from cpython.datetime cimport datetime, total_seconds
+from cpython.datetime cimport date_from_timestamp, get_utc, datetime_from_timestamp
+
+# These were added in Python 2.7.5, make sure that their backport works.
+from cpython.datetime cimport (
+    timedelta as timedelta_ext_type,
+    PyDateTime_DELTA_GET_DAYS,
+    PyDateTime_DELTA_GET_SECONDS,
+    PyDateTime_DELTA_GET_MICROSECONDS,
+)
 
 import datetime as py_datetime
+import time as py_time
+import sys
 
 import_datetime()
 
@@ -37,7 +47,23 @@
 
     def dst(self, dt):
         return ZERO
-        
+
+
+def do_timedelta_macros(timedelta_ext_type delta):
+    """
+    >>> delta = py_datetime.timedelta(days=13, hours=7, seconds=31, microseconds=993322)
+    >>> (delta.days, delta.seconds, delta.microseconds)
+    (13, 25231, 993322)
+    >>> do_timedelta_macros(delta)
+    (13, 25231, 993322)
+    """
+    return (
+        PyDateTime_DELTA_GET_DAYS(delta),
+        PyDateTime_DELTA_GET_SECONDS(delta),
+        PyDateTime_DELTA_GET_MICROSECONDS(delta),
+    )
+
+
 def do_date(int year, int month, int day):
     """
     >>> do_date(2012, 12, 31)
@@ -46,7 +72,7 @@
     v = date_new(year, month, day)
     return type(v) is py_datetime.date, v.year == year, v.month == month, v.day == day
 
-def do_datetime(int year, int month, int day, 
+def do_datetime(int year, int month, int day,
         int hour, int minute, int second, int microsecond):
     """
     >>> do_datetime(2012, 12, 31, 12, 23, 0, 0)
@@ -69,7 +95,7 @@
 
 def do_time_tzinfo(int hour, int minute, int second, int microsecond, object tz):
     """
-    >>> tz = FixedOffset(60*3, 'Moscow')    
+    >>> tz = FixedOffset(60*3, 'Moscow')
     >>> do_time_tzinfo(12, 23, 0, 0, tz)
     (True, True, True, True, True, True)
     """
@@ -79,10 +105,10 @@
            v.microsecond == microsecond, v.tzinfo is tz
 
 
-def do_datetime_tzinfo(int year, int month, int day, 
+def do_datetime_tzinfo(int year, int month, int day,
         int hour, int minute, int second, int microsecond, object tz):
     """
-    >>> tz = FixedOffset(60*3, 'Moscow')    
+    >>> tz = FixedOffset(60*3, 'Moscow')
     >>> do_datetime_tzinfo(2012, 12, 31, 12, 23, 0, 0, tz)
     (True, True, True, True, True, True, True, True, True)
     """
@@ -90,35 +116,35 @@
     return type(v) is py_datetime.datetime, v.year == year, v.month == month, v.day == day, \
            v.hour == hour, v.minute == minute, v.second == second, \
            v.microsecond == microsecond, v.tzinfo is tz
-           
+
 def do_time_tzinfo2(int hour, int minute, int second, int microsecond, object tz):
     """
-    >>> tz = FixedOffset(60*3, 'Moscow')    
+    >>> tz = FixedOffset(60*3, 'Moscow')
     >>> do_time_tzinfo2(12, 23, 0, 0, tz)
     (True, True, True, True, True, True, True, True)
     """
     v = time_new(hour, minute, second, microsecond, None)
     v1 = time_new(
-            time_hour(v), 
-            time_minute(v), 
-            time_second(v), 
-            time_microsecond(v), 
+            time_hour(v),
+            time_minute(v),
+            time_second(v),
+            time_microsecond(v),
             tz)
     r1 = (v1.tzinfo == tz)
     r2 = (tz == time_tzinfo(v1))
     v2 = time_new(
-            time_hour(v1), 
-            time_minute(v1), 
-            time_second(v1), 
-            time_microsecond(v1), 
+            time_hour(v1),
+            time_minute(v1),
+            time_second(v1),
+            time_microsecond(v1),
             None)
     r3 = (v2.tzinfo == None)
     r4 = (None == time_tzinfo(v2))
     v3 = time_new(
-            time_hour(v2), 
-            time_minute(v2), 
-            time_second(v2), 
-            time_microsecond(v2), 
+            time_hour(v2),
+            time_minute(v2),
+            time_second(v2),
+            time_microsecond(v2),
             tz)
     r5 = (v3.tzinfo == tz)
     r6 = (tz == time_tzinfo(v3))
@@ -130,44 +156,124 @@
 def do_datetime_tzinfo2(int year, int month, int day,
                               int hour, int minute, int second, int microsecond, object tz):
     """
-    >>> tz = FixedOffset(60*3, 'Moscow')    
+    >>> tz = FixedOffset(60*3, 'Moscow')
     >>> do_datetime_tzinfo2(2012, 12, 31, 12, 23, 0, 0, tz)
     (True, True, True, True, True, True, True, True)
     """
     v = datetime_new(year, month, day, hour, minute, second, microsecond, None)
     v1 = datetime_new(
-            datetime_year(v), 
-            datetime_month(v), 
-            datetime_day(v), 
-            datetime_hour(v), 
-            datetime_minute(v), 
-            datetime_second(v), 
-            datetime_microsecond(v), 
+            datetime_year(v),
+            datetime_month(v),
+            datetime_day(v),
+            datetime_hour(v),
+            datetime_minute(v),
+            datetime_second(v),
+            datetime_microsecond(v),
             tz)
     r1 = (v1.tzinfo == tz)
     r2 = (tz == datetime_tzinfo(v1))
     v2 = datetime_new(
-            datetime_year(v1), 
-            datetime_month(v1), 
-            datetime_day(v1), 
-            datetime_hour(v1), 
-            datetime_minute(v1), 
-            datetime_second(v1), 
-            datetime_microsecond(v1), 
+            datetime_year(v1),
+            datetime_month(v1),
+            datetime_day(v1),
+            datetime_hour(v1),
+            datetime_minute(v1),
+            datetime_second(v1),
+            datetime_microsecond(v1),
             None)
     r3 = (v2.tzinfo == None)
     r4 = (None == datetime_tzinfo(v2))
     v3 = datetime_new(
-            datetime_year(v2), 
-            datetime_month(v2), 
-            datetime_day(v2), 
-            datetime_hour(v2), 
-            datetime_minute(v2), 
-            datetime_second(v2), 
-            datetime_microsecond(v2), 
+            datetime_year(v2),
+            datetime_month(v2),
+            datetime_day(v2),
+            datetime_hour(v2),
+            datetime_minute(v2),
+            datetime_second(v2),
+            datetime_microsecond(v2),
             tz)
     r5 = (v3.tzinfo == tz)
     r6 = (tz == datetime_tzinfo(v3))
     r7 = (v2 == v)
     r8 = (v3 == v1)
     return r1, r2, r3, r4, r5, r6, r7, r8
+
+
+def test_timedelta_total_seconds():
+    """
+    >>> cytotal, pytotal = test_timedelta_total_seconds()
+    >>> assert cytotal == pytotal, (cytotal, pytotal)
+    >>> cytotal == pytotal
+    True
+    """
+    cdef:
+        datetime now = py_datetime.datetime.now()
+        timedelta td = now - py_datetime.datetime(1970, 1, 1)
+
+    pytd = now - py_datetime.datetime(1970, 1, 1)
+
+    return total_seconds(td), pytd.total_seconds()
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceFromPyTypeNode",
+    "//AttributeNode",
+)
+def test_datetime_attrs_inlined(datetime dt):
+    # GH#3737
+    """
+    >>> from datetime import datetime
+    >>> py_dt = datetime(2020, 8, 18, 4, 9)
+    >>> dt = test_datetime_attrs_inlined(py_dt)
+    >>> dt[:5]
+    (2020, 8, 18, 4, 9)
+    >>> dt[5] == py_dt.second  or  (dt[5], py_dt.second)
+    True
+    >>> dt[6] == py_dt.microsecond  or  (dt[6], py_dt.microsecond)
+    True
+    """
+    return (
+        dt.year,
+        dt.month,
+        dt.day,
+        dt.hour,
+        dt.minute,
+        dt.second,
+        dt.microsecond,
+    )
+
+def test_date_from_timestamp():
+    """
+    >>> from datetime import datetime
+    >>> tp, dt = test_date_from_timestamp()
+    >>> tp == dt
+    True
+    """
+    tp = date_from_timestamp(1518185542)
+    dt = py_datetime.date(2018, 2, 9)
+    return tp, dt
+
+def test_get_utc():
+    """
+    >>> from datetime import datetime
+    >>> test_get_utc()
+    True
+    """
+    try:
+        get_utc()
+    except RuntimeError:
+        if sys.version_info >= (3, 7):
+            raise  # get_utc() is only supposed to raise on Python < 3.7
+    return True
+
+def test_datetime_from_timestamp():
+    """
+    >>> from datetime import datetime
+    >>> tp, dt = test_datetime_from_timestamp()
+    >>> tp == dt
+    True
+    """
+    time = py_time.time()
+    tp = datetime_from_timestamp(time)
+    dt = py_datetime.datetime.fromtimestamp(time)
+    return tp, dt
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/decorators_py_T593.py cython-0.20.1+1~202203241016-9537/tests/run/decorators_py_T593.py
--- cython-0.20.1+1~201611251650-6686/tests/run/decorators_py_T593.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/decorators_py_T593.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 593
+# ticket: t593
 # tag: property, decorator
 
 """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/decorators.pyx cython-0.20.1+1~202203241016-9537/tests/run/decorators.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/decorators.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/decorators.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -61,3 +61,23 @@
 @a.decorate
 def i(x):
     return x - 1
+
+def append_to_list_decorator(lst):
+    def do_append_to_list_dec(func):
+        def new_func():
+            return lst + func()
+        return new_func
+    return do_append_to_list_dec
+
+def outer(arg1, arg2):
+    """
+    ensure decorators are analysed in the correct scope
+    https://github.com/cython/cython/issues/4367
+    mainly intended as a compile-time test (but it does run...)
+    >>> outer(append_to_list_decorator, [1,2,3])
+    [1, 2, 3, 4]
+    """
+    @arg1([x for x in arg2])
+    def method():
+        return [4]
+    return method()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/decorators_T593.pyx cython-0.20.1+1~202203241016-9537/tests/run/decorators_T593.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/decorators_T593.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/decorators_T593.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 593
+# ticket: t593
 # tag: property, decorator
 
 """
@@ -110,8 +110,8 @@
 
 class Bar(metaclass=Base):
    """
-   >>> Bar._order
-   ['__module__', '__qualname__', '__doc__', 'bar']
+   >>> [n for n in Bar._order if n not in {"__qualname__", "__annotations__"}]
+   ['__module__', '__doc__', 'bar']
    """
    @property
    def bar(self):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/default_args_T674.py cython-0.20.1+1~202203241016-9537/tests/run/default_args_T674.py
--- cython-0.20.1+1~201611251650-6686/tests/run/default_args_T674.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/default_args_T674.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 674
+# ticket: t674
 
 def test_inner(a):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/dictcomp.pyx cython-0.20.1+1~202203241016-9537/tests/run/dictcomp.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/dictcomp.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/dictcomp.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -57,3 +57,35 @@
     l = list(it)
     l.sort()
     return l
+
+
+# Copied from sre_compile.py in CPython 3.7.  Previously failed to detect variable initialisation.
+_equivalences = (
+    # LATIN SMALL LETTER I, LATIN SMALL LETTER DOTLESS I
+    (0x69, 0x131), # iı
+    # LATIN SMALL LETTER S, LATIN SMALL LETTER LONG S
+    (0x73, 0x17f), # sſ
+    # MICRO SIGN, GREEK SMALL LETTER MU
+    (0xb5, 0x3bc), # µμ
+    # COMBINING GREEK YPOGEGRAMMENI, GREEK SMALL LETTER IOTA, GREEK PROSGEGRAMMENI
+    (0x345, 0x3b9, 0x1fbe), # \u0345ιι
+    # ...
+)
+
+_ignorecase_fixes = {
+    i: tuple(j for j in t if i != j)
+    for t in _equivalences for i in t
+}
+
+def nested_tuple():
+    """
+    >>> modlevel, funclevel = nested_tuple()
+    >>> modlevel == funclevel or (modlevel, funclevel)
+    True
+    """
+    inner_ignorecase_fixes = {
+        i: tuple(j for j in t if i != j)
+        for t in _equivalences for i in t
+    }
+
+    return sorted(_ignorecase_fixes.items()), sorted(inner_ignorecase_fixes.items())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/dict_getitem.pyx cython-0.20.1+1~202203241016-9537/tests/run/dict_getitem.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/dict_getitem.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/dict_getitem.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -46,6 +46,72 @@
     """
     return d[index]
 
+
+def getitem_str(dict d, obj, str index):
+    """
+    >>> d = {'abc': 1, 'xyz': 2, None: 3}
+    >>> getitem_str(d, d, 'abc')
+    (1, 1)
+    >>> getitem_str(d, d, 'xyz')
+    (2, 2)
+    >>> getitem_str(d, d, None)
+    (3, 3)
+
+    >>> class GetItem(object):
+    ...     def __getitem__(self, name): return d[name]
+    >>> getitem_str(d, GetItem(), 'abc')
+    (1, 1)
+    >>> getitem_str(d, GetItem(), 'xyz')
+    (2, 2)
+    >>> getitem_str(d, GetItem(), None)
+    (3, 3)
+    >>> getitem_str(d, GetItem(), 'no')
+    Traceback (most recent call last):
+    KeyError: 'no'
+
+    >>> class GetItemFail(object):
+    ...     def __getitem__(self, name): raise ValueError("failed")
+    >>> getitem_str(d, GetItemFail(), 'abc')
+    Traceback (most recent call last):
+    ValueError: failed
+    >>> getitem_str(d, GetItemFail(), None)
+    Traceback (most recent call last):
+    ValueError: failed
+    """
+    return d[index], obj[index]
+
+
+def getitem_unicode(dict d, obj, unicode index):
+    """
+    >>> d = {'abc': 1, 'xyz': 2, None: 3}
+    >>> getitem_unicode(d, d, u'abc')
+    (1, 1)
+    >>> getitem_unicode(d, d, u'xyz')
+    (2, 2)
+    >>> getitem_unicode(d, d, None)
+    (3, 3)
+
+    >>> class GetItem(object):
+    ...     def __getitem__(self, name): return d[name]
+    >>> getitem_unicode(d, GetItem(), u'abc')
+    (1, 1)
+    >>> getitem_unicode(d, GetItem(), u'xyz')
+    (2, 2)
+    >>> getitem_unicode(d, GetItem(), None)
+    (3, 3)
+    >>> try: getitem_unicode(d, GetItem(), u'no')
+    ... except KeyError as exc: assert exc.args[0] == u'no', str(exc)
+    ... else: assert False, "KeyError not raised"
+
+    >>> class GetItemFail(object):
+    ...     def __getitem__(self, name): raise ValueError("failed")
+    >>> getitem_unicode(d, GetItemFail(), u'abc')
+    Traceback (most recent call last):
+    ValueError: failed
+    """
+    return d[index], obj[index]
+
+
 def getitem_tuple(dict d, index):
     """
     >>> d = {1: 1, (1,): 2}
@@ -54,6 +120,7 @@
     """
     return d[index], d[index,]
 
+
 def getitem_in_condition(dict d, key, expected_result):
     """
     >>> d = dict(a=1, b=2)
@@ -62,6 +129,7 @@
     """
     return d[key] is expected_result or d[key] == expected_result
 
+
 @cython.test_fail_if_path_exists('//NoneCheckNode')
 def getitem_not_none(dict d not None, key):
     """
@@ -78,3 +146,17 @@
     KeyError: (1, 2)
     """
     return d[key]
+
+
+def getitem_int_key(d, int key):
+    """
+    >>> d = {-1: 10}
+    >>> getitem_int_key(d, -1)  # dict
+    10
+    >>> class D(dict): pass
+    >>> d = D({-1: 10})
+    >>> getitem_int_key(d, -1)  # D
+    10
+    """
+    # Based on GH-1807: must check Mapping protocol first, even for integer "index" keys.
+    return d[key]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/dict_pop.pyx cython-0.20.1+1~202203241016-9537/tests/run/dict_pop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/dict_pop.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/dict_pop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,58 @@
+# mode: run
+# tag: dict, pop, builtins
+
+cimport cython
+
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+@cython.test_fail_if_path_exists("//AttributeNode")
+def dict_pop(dict d, key):
+    """
+    >>> d = { 1: 10, 2: 20 }
+    >>> dict_pop(d, 1)
+    (10, {2: 20})
+    >>> dict_pop(d, 3)
+    Traceback (most recent call last):
+    KeyError: 3
+    >>> dict_pop(d, 2)
+    (20, {})
+    """
+    return d.pop(key), d
+
+
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+@cython.test_fail_if_path_exists("//AttributeNode")
+def dict_pop_default(dict d, key, default):
+    """
+    >>> d = { 1: 10, 2: 20 }
+    >>> dict_pop_default(d, 1, "default")
+    (10, {2: 20})
+    >>> dict_pop_default(d, 3, None)
+    (None, {2: 20})
+    >>> dict_pop_default(d, 3, "default")
+    ('default', {2: 20})
+    >>> dict_pop_default(d, 2, "default")
+    (20, {})
+    """
+    return d.pop(key, default), d
+
+
+cdef class MyType:
+    cdef public int i
+    def __init__(self, i):
+        self.i = i
+
+
+@cython.test_assert_path_exists("//SingleAssignmentNode//PythonCapiCallNode")
+@cython.test_fail_if_path_exists("//SingleAssignmentNode//AttributeNode")
+def dict_pop_default_typed(dict d, key, default):
+    """
+    >>> d = {1: MyType(2)}
+    >>> dict_pop_default_typed(d, 1, None)
+    2
+    >>> dict_pop_default_typed(d, 3, None)
+    >>> dict_pop_default_typed(d, 3, "default")  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: Cannot convert str to ...MyType
+    """
+    cdef MyType x = d.pop(key, default)
+    return x.i if x is not None else None
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/dict.pyx cython-0.20.1+1~202203241016-9537/tests/run/dict.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/dict.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/dict.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -117,3 +117,22 @@
     [2, 4, 5]
     """
     return {1:2, sideeffect(2): 3, 3: 4, unhashable(4): 5, sideeffect(5): 6}
+
+
+def dict_unpacking_not_for_arg_create_a_copy():
+    """
+    >>> dict_unpacking_not_for_arg_create_a_copy()
+    [('a', 'modified'), ('b', 'original')]
+    [('a', 'original'), ('b', 'original')]
+    """
+    data = {'a': 'original', 'b': 'original'}
+
+    func = lambda: {**data}
+
+    call_once = func()
+    call_once['a'] = 'modified'
+
+    call_twice = func()
+
+    print(sorted(call_once.items()))
+    print(sorted(call_twice.items()))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/different_package_names.srctree cython-0.20.1+1~202203241016-9537/tests/run/different_package_names.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/different_package_names.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/different_package_names.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,43 @@
+# mode: run
+# tag: import,cimport,packages
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import pkg_py"
+PYTHON -c "import pkg_py.pkg_pyx"
+PYTHON -c "import pkg_py.pkg_pyx.module as module; module.run_test()"
+
+######## setup.py ########
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(
+    ext_modules=cythonize('**/*.pyx', language_level=3),
+)
+
+
+######## pkg_py/__init__.py ########
+
+TYPE = 'py'
+
+######## pkg_py/pkg_pyx/__init__.pyx ########
+
+TYPE = 'pyx'
+
+######## pkg_py/pkg_pyx/pkg_pxd/__init__.pxd ########
+
+# Not what Python would consider a package, but Cython can use it for cimports.
+from libc.math cimport fabs
+
+######## pkg_py/pkg_pyx/module.pyx ########
+
+from pkg_py.pkg_pyx.pkg_pxd cimport fabs
+
+def run_test():
+    import pkg_py
+    assert pkg_py.TYPE == 'py'
+
+    import pkg_py.pkg_pyx
+    assert pkg_py.pkg_pyx.TYPE == 'pyx'
+
+    assert fabs(-2.0) == 2.0
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/division_T384.pyx cython-0.20.1+1~202203241016-9537/tests/run/division_T384.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/division_T384.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/division_T384.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 384
+# ticket: t384
 
 """
 >>> test(3)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/duplicate_keyword_in_call.py cython-0.20.1+1~202203241016-9537/tests/run/duplicate_keyword_in_call.py
--- cython-0.20.1+1~201611251650-6686/tests/run/duplicate_keyword_in_call.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/duplicate_keyword_in_call.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: kwargs, call
-# ticket: 717
+# ticket: t717
 
 def f(**kwargs):
     return sorted(kwargs.items())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/duplicate_utilitycode_from_pyx.srctree cython-0.20.1+1~202203241016-9537/tests/run/duplicate_utilitycode_from_pyx.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/duplicate_utilitycode_from_pyx.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/duplicate_utilitycode_from_pyx.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,29 @@
+
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import modb; modb.ClassB()"
+
+#################### moda.pyx ####################
+
+cdef class ClassA:
+    cdef int[2] a
+
+#################### modb.pyx #####################
+
+from moda cimport ClassA
+
+cdef class ClassB(ClassA):
+    cdef int[2] b
+
+###################### setup.py ###################
+
+from setuptools import setup
+from Cython.Build import cythonize
+import Cython.Compiler.Options
+
+Cython.Compiler.Options.cimport_from_pyx = True
+
+setup(
+        ext_modules = cythonize(["moda.pyx", "modb.pyx"],
+            compiler_directives={'language_level': 3})
+)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/dynamic_args.pyx cython-0.20.1+1~202203241016-9537/tests/run/dynamic_args.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/dynamic_args.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/dynamic_args.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 674
+# ticket: t674
 
 cdef class Foo:
     cdef str name
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ellipsis_T488.pyx cython-0.20.1+1~202203241016-9537/tests/run/ellipsis_T488.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ellipsis_T488.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ellipsis_T488.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 488
+# ticket: t488
 
 """
 >>> test()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/embedsignatures.pyx cython-0.20.1+1~202203241016-9537/tests/run/embedsignatures.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/embedsignatures.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/embedsignatures.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,10 +1,16 @@
-#cython: embedsignature=True
+#cython: embedsignature=True, annotation_typing=False
+
+# signatures here are a little fragile - when they are
+# generated during the build process gives slightly
+# different (but equivalent) forms - therefore tests
+# may need changing occasionally to reflect behaviour
+# and this isn't necessarily a bug
 
 import sys
 
 if sys.version_info >= (3, 4):
     def funcdoc(f):
-        if not f.__text_signature__:
+        if not getattr(f, "__text_signature__", None):
             return f.__doc__
         doc = '%s%s' % (f.__name__, f.__text_signature__)
         if f.__doc__:
@@ -80,6 +86,12 @@
     >>> print (Ext.m.__doc__)
     Ext.m(self, a=u'spam')
 
+    >>> print (Ext.n.__doc__)
+    Ext.n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True)
+
+    >>> print (Ext.o.__doc__)
+    Ext.o(self, a, b=1, /, c=5, *args, **kwargs)
+
     >>> print (Ext.get_int.__doc__)
     Ext.get_int(self) -> int
 
@@ -185,7 +197,7 @@
     f_defexpr4(int x=(Ext.CONST1 + FLAG1) * Ext.CONST2)
 
     >>> print(funcdoc(f_defexpr5))
-    f_defexpr5(int x=4)
+    f_defexpr5(int x=2 + 2)
 
     >>> print(funcdoc(f_charptr_null))
     f_charptr_null(char *s=NULL) -> char *
@@ -259,6 +271,12 @@
     def m(self, a=u'spam'):
         pass
 
+    def n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True):
+        pass
+
+    def o(self, a, b=1, /, c=5, *args, **kwargs):
+        pass
+
     cpdef int get_int(self):
         return 0
 
@@ -388,3 +406,136 @@
 # no signatures for lambda functions
 lambda_foo = lambda x: 10
 lambda_bar = lambda x: 20
+
+
+cdef class Foo:
+    def m00(self, a: None) ->  None: pass
+    def m01(self, a: ...) ->  Ellipsis: pass
+    def m02(self, a: True, b: False) ->  bool: pass
+    def m03(self, a: 42, b: +42, c: -42) ->  int : pass  # XXX +42 -> 42
+    def m04(self, a: 3.14, b: +3.14, c: -3.14) -> float : pass
+    def m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex : pass
+    def m06(self, a: "abc", b: b"abc", c: u"abc") -> (str, bytes, unicode) : pass
+    def m07(self, a: [1, 2, 3], b: []) -> list: pass
+    def m08(self, a: (1, 2, 3), b: ()) -> tuple: pass
+    def m09(self, a: {1, 2, 3}, b: {i for i in ()}) -> set: pass
+    def m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict: pass
+   #def m11(self, a: [str(i) for i in range(3)]): pass  # Issue 1782
+    def m12(self, a: (str(i) for i in range(3))): pass
+    def m13(self, a: (str(i) for i in range(3) if bool(i))): pass
+    def m14(self, a: {str(i) for i in range(3)}): pass
+    def m15(self, a: {str(i) for i in range(3) if bool(i)}): pass
+    def m16(self, a: {str(i): id(i) for i in range(3)}): pass
+    def m17(self, a: {str(i): id(i) for i in range(3) if bool(i)}): pass
+    def m18(self, a: dict.update(x=42, **dict(), **{})): pass
+    def m19(self, a: sys is None, b: sys is not None): pass
+    def m20(self, a: sys in [], b: sys not in []): pass
+    def m21(self, a: (sys or sys) and sys, b: not (sys or sys)): pass
+    def m22(self, a: 42 if sys else None): pass
+    def m23(self, a: +int(), b: -int(), c: ~int()): pass
+    def m24(self, a: (1+int(2))*3+(4*int(5))**(1+0.0/1)): pass
+    def m25(self, a: list(range(3))[:]): pass
+    def m26(self, a: list(range(3))[1:]): pass
+    def m27(self, a: list(range(3))[:1]): pass
+    def m28(self, a: list(range(3))[::1]): pass
+    def m29(self, a: list(range(3))[0:1:1]): pass
+    def m30(self, a: list(range(3))[7, 3:2:1, ...]): pass
+    def m31(self, double[::1] a): pass
+
+__doc__ += ur"""
+>>> print(Foo.m00.__doc__)
+Foo.m00(self, a: None) -> None
+
+>>> print(Foo.m01.__doc__)
+Foo.m01(self, a: ...) -> Ellipsis
+
+>>> print(Foo.m02.__doc__)
+Foo.m02(self, a: True, b: False) -> bool
+
+>>> print(Foo.m03.__doc__)
+Foo.m03(self, a: 42, b: +42, c: -42) -> int
+
+>>> print(Foo.m04.__doc__)
+Foo.m04(self, a: 3.14, b: +3.14, c: -3.14) -> float
+
+>>> print(Foo.m05.__doc__)
+Foo.m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex
+
+>>> print(Foo.m06.__doc__)
+Foo.m06(self, a: 'abc', b: b'abc', c: 'abc') -> (str, bytes, unicode)
+
+>>> print(Foo.m07.__doc__)
+Foo.m07(self, a: [1, 2, 3], b: []) -> list
+
+>>> print(Foo.m08.__doc__)
+Foo.m08(self, a: (1, 2, 3), b: ()) -> tuple
+
+>>> print(Foo.m09.__doc__)
+Foo.m09(self, a: {1, 2, 3}, b: {i for i in ()}) -> set
+
+>>> print(Foo.m10.__doc__)
+Foo.m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict
+
+# >>> print(Foo.m11.__doc__)
+# Foo.m11(self, a: [str(i) for i in range(3)])
+
+>>> print(Foo.m12.__doc__)
+Foo.m12(self, a: (str(i) for i in range(3)))
+
+>>> print(Foo.m13.__doc__)
+Foo.m13(self, a: (str(i) for i in range(3) if bool(i)))
+
+>>> print(Foo.m14.__doc__)
+Foo.m14(self, a: {str(i) for i in range(3)})
+
+>>> print(Foo.m15.__doc__)
+Foo.m15(self, a: {str(i) for i in range(3) if bool(i)})
+
+>>> print(Foo.m16.__doc__)
+Foo.m16(self, a: {str(i): id(i) for i in range(3)})
+
+>>> print(Foo.m17.__doc__)
+Foo.m17(self, a: {str(i): id(i) for i in range(3) if bool(i)})
+
+>>> print(Foo.m18.__doc__)
+Foo.m18(self, a: dict.update(x=42, **dict()))
+
+>>> print(Foo.m19.__doc__)
+Foo.m19(self, a: sys is None, b: sys is not None)
+
+>>> print(Foo.m20.__doc__)
+Foo.m20(self, a: sys in [], b: sys not in [])
+
+>>> print(Foo.m21.__doc__)
+Foo.m21(self, a: (sys or sys) and sys, b: not (sys or sys))
+
+>>> print(Foo.m22.__doc__)
+Foo.m22(self, a: 42 if sys else None)
+
+>>> print(Foo.m23.__doc__)
+Foo.m23(self, a: +int(), b: -int(), c: ~int())
+
+>>> print(Foo.m24.__doc__)
+Foo.m24(self, a: (1 + int(2)) * 3 + (4 * int(5)) ** (1 + 0.0 / 1))
+
+>>> print(Foo.m25.__doc__)
+Foo.m25(self, a: list(range(3))[:])
+
+>>> print(Foo.m26.__doc__)
+Foo.m26(self, a: list(range(3))[1:])
+
+>>> print(Foo.m27.__doc__)
+Foo.m27(self, a: list(range(3))[:1])
+
+>>> print(Foo.m28.__doc__)
+Foo.m28(self, a: list(range(3))[::1])
+
+>>> print(Foo.m29.__doc__)
+Foo.m29(self, a: list(range(3))[0:1:1])
+
+>>> print(Foo.m30.__doc__)
+Foo.m30(self, a: list(range(3))[7, 3:2:1, ...])
+
+>>> print(Foo.m31.__doc__)
+Foo.m31(self, double[::1] a)
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/empty_for_loop_T208.pyx cython-0.20.1+1~202203241016-9537/tests/run/empty_for_loop_T208.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/empty_for_loop_T208.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/empty_for_loop_T208.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 208
+# ticket: t208
 
 def go_py_empty():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/enumerate_T316.pyx cython-0.20.1+1~202203241016-9537/tests/run/enumerate_T316.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/enumerate_T316.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/enumerate_T316.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 316
+# ticket: t316
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/error_pos.srctree cython-0.20.1+1~202203241016-9537/tests/run/error_pos.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/error_pos.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/error_pos.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,24 @@
+PYTHON setup.py build_ext --inplace
+PYTHON test_error_pos.py
+
+######## setup.py ###########
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules=cythonize("error_pos.pyx"))
+
+######## error_pos.pyx ###########
+from os import *
+
+abcdefg(line)
+
+######## test_error_pos.py ###########
+import subprocess
+import sys
+
+cmd = [sys.executable, '-c', 'import error_pos']
+proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
+_, err = proc.communicate()
+# The error should contain the line number and the line text where the
+# undefined identifier is used.
+assert b'line 3, in init error_pos' and b'abcdefg(line)' in err, err
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exceptionpropagation.pyx cython-0.20.1+1~202203241016-9537/tests/run/exceptionpropagation.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exceptionpropagation.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exceptionpropagation.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -42,3 +42,18 @@
     RuntimeError
     """
     except_big_result(fire)
+
+
+cdef unsigned short except_promotion_compare(bint fire) except *:
+    if fire:
+        raise RuntimeError
+
+def test_except_promotion_compare(bint fire):
+    """
+    >>> test_except_promotion_compare(False)
+    >>> test_except_promotion_compare(True)
+    Traceback (most recent call last):
+    ...
+    RuntimeError
+    """
+    except_promotion_compare(fire)
\ No newline at end of file
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exceptionrefcount.pyx cython-0.20.1+1~202203241016-9537/tests/run/exceptionrefcount.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exceptionrefcount.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exceptionrefcount.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,6 @@
-__doc__ = u"""
+# mode: run
+
+"""
 >>> class SampleException(Exception): pass
 
 >>> def assert_refcount(rc1, rc2, func):
@@ -27,8 +29,11 @@
 >>> run_test(50, test_finally)
 """
 
+cimport cython
 from cpython.ref cimport PyObject
 
+@cython.binding(False)
+@cython.always_allow_keywords(False)
 def get_refcount(obj):
     return (obj).ob_refcnt
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exceptions_nogil.pyx cython-0.20.1+1~202203241016-9537/tests/run/exceptions_nogil.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exceptions_nogil.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exceptions_nogil.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,14 +1,125 @@
+# mode: run
+# tag: nogil, withgil, exceptions
+
+cdef void foo_nogil(int i) nogil except *:
+    if i != 0: raise ValueError("huhu !")
+
+
 cdef void foo(int i) except * with gil:
     if i != 0: raise ValueError
 
+
 cdef int bar(int i) except? -1 with gil:
     if i != 0: raise ValueError
     return 0
 
+
 cdef int spam(int i) except? -1 with gil:
     if i != 0: raise TypeError
     return -1
 
+
+def test_foo_nogil():
+    """
+    >>> test_foo_nogil()
+    """
+    #
+    foo_nogil(0)
+    foo_nogil(0)
+    with nogil:
+        foo_nogil(0)
+        foo_nogil(0)
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+    finally:
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+        with nogil:
+            foo_nogil(0)
+    finally:
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+        with nogil:
+            foo_nogil(1)
+    except:
+        with nogil:
+            foo_nogil(0)
+    finally:
+        with nogil:
+            foo_nogil(0)
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+            foo_nogil(0)
+    finally:
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+            foo_nogil(1)
+    except:
+        with nogil:
+            foo_nogil(0)
+    finally:
+        with nogil:
+            foo_nogil(0)
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+        try:
+            with nogil:
+                foo_nogil(1)
+        except:
+            with nogil:
+                foo_nogil(1)
+        finally:
+            with nogil:
+                foo_nogil(0)
+            pass
+    except:
+        with nogil:
+            foo_nogil(0)
+    finally:
+        with nogil:
+            foo_nogil(0)
+        pass
+    #
+    try:
+        with nogil:
+            foo_nogil(0)
+        try:
+            with nogil:
+                foo_nogil(1)
+        except:
+            with nogil:
+                foo_nogil(1)
+        finally:
+            with nogil:
+                foo_nogil(1)
+            pass
+    except:
+        with nogil:
+            foo_nogil(0)
+    finally:
+        with nogil:
+            foo_nogil(0)
+        pass
+    #
+
+
 def test_foo():
     """
     >>> test_foo()
@@ -109,6 +220,7 @@
         pass
     #
 
+
 def test_bar():
     """
     >>> test_bar()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exectest.pyx cython-0.20.1+1~202203241016-9537/tests/run/exectest.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exectest.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exectest.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -145,3 +145,18 @@
     TypeError: exec: arg 1 must be string, bytes or code object, got int
     """
     exec x in {}
+
+
+def exec_with_new_features(s, d):
+    """
+    >>> import sys
+    >>> pyversion = sys.version_info[:2]
+
+    >>> d = {}
+    >>> exec_with_new_features('print(123)', d)
+    123
+    >>> if pyversion == (2, 7): exec_with_new_features('exec "123"', d)
+    >>> if pyversion >= (3, 6): exec_with_new_features('f = f"abc"', d)
+    >>> if pyversion >= (3, 8): exec_with_new_features('a = (b := 1)', d)
+    """
+    exec s in d
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/existing_output_files.srctree cython-0.20.1+1~202203241016-9537/tests/run/existing_output_files.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/existing_output_files.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/existing_output_files.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,179 @@
+PYTHON test.py
+
+######## test.py ########
+
+from __future__ import print_function
+
+import os.path
+from Cython.Utils import is_cython_generated_file
+from Cython.Compiler.Errors import CompileError
+from Cython.Build.Dependencies import cythonize
+
+# Make sure the source files are newer than the .c files, so that cythonize() regenerates them.
+files = {}
+for source_file in sorted(os.listdir(os.getcwd())):
+    if 'module' in source_file and (source_file.endswith(".pyx") or source_file.endswith(".py")):
+        c_file = files[source_file] = os.path.splitext(source_file)[0] + ".c"
+        os.utime(source_file, None)
+        assert not os.path.exists(c_file) or os.path.getmtime(source_file) >= os.path.getmtime(c_file)
+
+for source_file, c_file in files.items():
+    print("Testing:", source_file, c_file)
+    assert is_cython_generated_file(c_file, allow_failed=True, if_not_found=True)
+
+    # cythonizing should (re)generate the file
+    cythonize(source_file, language_level=3)
+    assert is_cython_generated_file(c_file, if_not_found=False)
+    assert os.path.getmtime(source_file) <= os.path.getmtime(c_file)
+
+    # calling cythonize again should not rewrite the file
+    # (not asserting this here, but at least it shouldn't fail)
+    cythonize(source_file, language_level=3)
+    assert is_cython_generated_file(c_file, if_not_found=False)
+    assert os.path.getmtime(source_file) <= os.path.getmtime(c_file)
+
+
+# But overwriting an unknown file should fail, even when requested multiple times.
+for source_file in [
+        "refuse_to_overwrite.pyx",
+        "refuse_to_overwrite.py",
+        "compile_failure.pyx",
+        "refuse_to_overwrite_header.pyx",
+        "refuse_to_overwrite_api_header.pyx",
+]:
+    if 'api_header' in source_file:
+        target_file = os.path.splitext(source_file)[0] + "_api.h"
+    elif 'header' in source_file:
+        target_file = os.path.splitext(source_file)[0] + ".h"
+    else:
+        target_file = os.path.splitext(source_file)[0] + ".c"
+
+    for _ in range(3):
+        os.utime(source_file, None)
+        assert not is_cython_generated_file(target_file)
+        try:
+            print("Testing:", source_file)
+            cythonize(source_file, language_level=3)
+        except CompileError:
+            print("REFUSED to overwrite %s, OK" % target_file)
+            assert not is_cython_generated_file(target_file)
+        else:
+            assert False, "FAILURE: Existing output file was overwritten for source file %s" % source_file
+
+
+######## pymodule.c ########
+#error Do not use this file, it is the result of a failed Cython compilation.
+
+######## pymodule.py ########
+"""
+Overwriting a failed .py file result works
+"""
+
+######## cymodule.c ########
+#error Do not use this file, it is the result of a failed Cython compilation.
+
+######## cymodule.pyx ########
+"""
+Overwriting a failed .pyx file result works
+"""
+
+######## overwritten_cymodule.c ########
+/* Generated by Cython 0.8.15 */
+
+######## overwritten_cymodule.pyx ########
+"""
+Overwriting an outdated .c file works
+"""
+
+
+######## new_cymodule.pyx ########
+"""
+Creating a new .c file works
+"""
+
+######## new_pymodule.py ########
+"""
+Creating a new .c file works
+"""
+
+
+######## refuse_to_overwrite.c ########
+static int external_function(int x) {
+    return x + 1;
+}
+
+######## refuse_to_overwrite.py ########
+"""
+Do not overwrite an unknown output file
+"""
+
+######## refuse_to_overwrite.pyx ########
+"""
+Do not overwrite an unknown output file
+"""
+
+
+######## compile_failure.c ########
+static int external_function(int x) {
+    return x + 1;
+}
+
+######## compile_failure.pyx ########
+"""
+Do not overwrite an unknown output file even on compile failures.
+"""
+
+Not Python syntax!
+
+
+
+######## write_module_header.pyx ########
+
+cdef public int func():
+    return 1
+
+
+######## overwrite_module_header.c ########
+/* Generated by Cython 0.8.15 */
+
+######## overwrite_module_header.pyx ########
+
+cdef public int func():
+    return 1
+
+
+######## refuse_to_overwrite_header.h ########
+static int external_function(int x) {
+    return x + 1;
+}
+
+######## refuse_to_overwrite_header.pyx ########
+
+cdef public int func():
+    return 1
+
+
+######## write_module_api_header.pyx ########
+
+cdef api int func():
+    return 1
+
+
+######## overwrite_module_api_header.c ########
+/* Generated by Cython 0.8.15 */
+
+######## overwrite_module_api_header.pyx ########
+
+cdef public int func():
+    return 1
+
+
+######## refuse_to_overwrite_api_header_api.h ########
+static int external_function(int x) {
+    return x + 1;
+}
+
+######## refuse_to_overwrite_api_header.pyx ########
+
+cdef api int func():
+    return 1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ext_attr_getter.srctree cython-0.20.1+1~202203241016-9537/tests/run/ext_attr_getter.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/ext_attr_getter.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ext_attr_getter.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,368 @@
+# mode: run
+# tag: cgetter, property
+
+"""
+PYTHON setup.py build_ext --inplace
+PYTHON run_failure_tests.py
+PYTHON runner.py
+"""
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+# Enforce the right build order
+setup(ext_modules = cythonize("foo_extension.pyx", language_level=3))
+setup(ext_modules = cythonize("getter[0-9].pyx", language_level=3))
+
+
+######## run_failure_tests.py ########
+
+import glob
+import sys
+
+from Cython.Build.Dependencies import cythonize
+from Cython.Compiler.Errors import CompileError
+
+# Run the failure tests
+failed_tests = []
+passed_tests = []
+
+def run_test(name):
+    title = name
+    with open(name, 'r') as f:
+        for line in f:
+            if 'TEST' in line:
+                title = line.partition('TEST:')[2].strip()
+                break
+    sys.stderr.write("\n### TESTING: %s\n" % title)
+
+    try:
+        cythonize(name, language_level=3)
+    except CompileError as e:
+        sys.stderr.write("\nOK: got expected exception\n")
+        passed_tests.append(name)
+    else:
+        sys.stderr.write("\nFAIL: compilation did not detect the error\n")
+        failed_tests.append(name)
+
+for name in sorted(glob.glob("getter_fail*.pyx")):
+    run_test(name)
+
+assert not failed_tests, "Failed tests: %s" % failed_tests
+assert passed_tests  # check that tests were found at all
+
+
+######## foo.h ########
+
+#include 
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f0;
+    int f1;
+    int f2;
+    int v[10];
+} FooStructNominal;
+
+typedef struct {
+    PyObject_HEAD
+} FooStructOpaque;
+
+
+#define PyFoo_GET0M(a) (((FooStructNominal*)a)->f0)
+#define PyFoo_GET1M(a) (((FooStructNominal*)a)->f1)
+#define PyFoo_GET2M(a) (((FooStructNominal*)a)->f2)
+
+int PyFoo_Get0F(FooStructOpaque *f)
+{
+    return PyFoo_GET0M(f);
+}
+
+int PyFoo_Get1F(FooStructOpaque *f)
+{
+    return PyFoo_GET1M(f);
+}
+
+int PyFoo_Get2F(FooStructOpaque *f)
+{
+    return PyFoo_GET2M(f);
+}
+
+int *PyFoo_GetV(FooStructOpaque *f)
+{
+    return ((FooStructNominal*)f)->v;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+
+######## foo_extension.pyx ########
+
+cdef class Foo:
+    cdef public int _field0, _field1, _field2;
+    cdef public int _vector[10];
+
+    @property
+    def field0(self):
+        return self._field0
+
+    @property
+    def field1(self):
+        return self._field1
+
+    @property
+    def field2(self):
+        return self._field2
+
+    def __init__(self, f0, f1, f2, vec=None):
+        if vec is None:
+            vec = ()
+        if not isinstance(vec, tuple):
+            raise ValueError("v must be None or a tuple")
+        self._field0 = f0
+        self._field1 = f1
+        self._field2 = f2
+        i = 0
+        for v in vec:
+            self._vector[i] = v
+            if i > 9:
+                break
+            i += 1
+        for j in range(i,10):
+            self._vector[j] = 0
+
+# A pure-python class that disallows direct access to fields
+class OpaqueFoo(Foo):
+
+    @property
+    def field0(self):
+        raise AttributeError('no direct access to field0')
+
+    @property
+    def field1(self):
+        raise AttributeError('no direct access to field1')
+
+    @property
+    def field2(self):
+        raise AttributeError('no direct access to field2')
+
+
+######## getter0.pyx ########
+
+# Access base Foo fields from C via aliased field names
+
+cdef extern from "foo.h":
+
+    ctypedef class foo_extension.Foo [object FooStructNominal]:
+        cdef:
+            int field0 "f0"
+            int field1 "f1"
+            int field2 "f2"
+
+def sum(Foo f):
+    # Note - not a cdef function but compiling the f.__getattr__('field0')
+    # notices the alias and replaces the __getattr__ in c by f->f0 anyway
+    return f.field0 + f.field1 + f.field2
+
+def check_pyobj(Foo f):
+    # compare the c code to the check_pyobj in getter2.pyx
+    return bool(f.field1)
+
+
+######## getter.pxd ########
+
+# Access base Foo fields from C via getter functions
+
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque, check_size ignore]:
+        @property
+        cdef inline int fieldM0(self):
+            return PyFoo_GET0M(self)
+
+        @property
+        cdef inline int fieldF1(self) except -123:
+            return PyFoo_Get1F(self)
+
+        @property
+        cdef inline int fieldM2(self):
+            return PyFoo_GET2M(self)
+
+        @property
+        cdef inline int *vector(self):
+            return PyFoo_GetV(self)
+
+        @property
+        cdef inline int meaning_of_life(self) except -99:
+            cdef int ret = 21
+            ret *= 2
+            return ret
+
+    int PyFoo_GET0M(Foo);  # this is actually a macro !
+    int PyFoo_Get1F(Foo);
+    int PyFoo_GET2M(Foo);  # this is actually a macro !
+    int *PyFoo_GetV(Foo);
+
+
+######## getter1.pyx ########
+
+cimport getter
+
+def sum(getter.Foo f):
+    # Note - not a cdef function but compiling the f.__getattr__('field0')
+    # notices the getter and replaces the __getattr__ in c by PyFoo_GET anyway
+    return f.fieldM0 + f.fieldF1 + f.fieldM2
+
+def check_10(getter.Foo f):
+    return f.fieldF1 != 10
+
+def vec0(getter.Foo f):
+    return f.vector[0]
+
+def check_binop(getter.Foo f):
+    return f.fieldF1 / 10
+
+
+######## getter2.pyx ########
+
+cimport getter
+
+def check_pyobj(getter.Foo f):
+    return bool(f.fieldF1)
+ 
+def check_unary(getter.Foo f):
+    return -f.fieldF1
+
+def check_meaning_of_life(getter.Foo f):
+    return f.meaning_of_life
+
+
+######## getter_fail_classmethod.pyx ########
+
+# TEST: Make sure not all decorators are accepted.
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        @classmethod
+        cdef inline int field0(cls):
+            print('in classmethod of Foo')
+
+
+######## getter_fail_dot_getter.pyx ########
+
+# TEST: Make sure not all decorators are accepted.
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        cdef inline int field0(self):
+            pass
+
+        @field0.getter
+        cdef inline void field1(self):
+            pass
+
+
+######## getter_fail_no_inline.pyx ########
+
+# TEST: Properties must be declared "inline".
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        cdef int field0(self):
+            pass
+
+
+######## getter_fail_void.pyx ########
+
+# TEST: Properties must have a non-void return type.
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        cdef void field0(self):
+            pass
+
+
+######## getter_fail_no_args.pyx ########
+
+# TEST: Properties must have the right signature.
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        cdef int field0():
+            pass
+
+
+######## getter_fail_too_many_args.pyx ########
+
+# TEST: Properties must have the right signature.
+
+cdef extern from "foo.h":
+    ctypedef class foo_extension.Foo [object FooStructOpaque]:
+        @property
+        cdef int field0(x, y):
+            pass
+
+
+######## runner.py ########
+
+import warnings
+import foo_extension, getter0, getter1, getter2
+
+def sum(f):
+    # pure python field access, but code is identical to cython cdef sum
+    return f.field0 + f.field1 + f.field2
+
+# Baseline test: if this fails something else is wrong
+foo = foo_extension.Foo(23, 123, 1023)
+
+assert foo.field0 == 23
+assert foo.field1 == 123
+assert foo.field2 == 1023
+
+ret =  getter0.sum(foo)
+assert ret == sum(foo)
+
+# Aliasing test. Check 'cdef int field0 "f0" works as advertised:
+# - C can access the fields through the aliases
+# - Python cannot access the fields at all
+
+opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023)
+
+opaque_ret = getter0.sum(opaque_foo)
+assert opaque_ret == ret
+
+val = getter2.check_pyobj(opaque_foo)
+assert val is True
+val = getter2.check_unary(opaque_foo)
+assert val == -123
+val = getter2.check_meaning_of_life(opaque_foo)
+assert val == 42
+
+try:
+    f0 = opaque_ret.field0
+    assert False
+except AttributeError as e:
+    pass
+
+# Getter test. Check C-level getter works as advertised:
+# - C accesses the fields through getter calls (maybe macros)
+# - Python accesses the fields through attribute lookup
+
+opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023, (1, 2, 3))
+
+opaque_ret = getter1.sum(opaque_foo)
+assert opaque_ret == ret
+
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ext_auto_richcmp.py cython-0.20.1+1~202203241016-9537/tests/run/ext_auto_richcmp.py
--- cython-0.20.1+1~201611251650-6686/tests/run/ext_auto_richcmp.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ext_auto_richcmp.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,581 @@
+# mode: run
+
+import cython
+compiled = cython.compiled
+
+import sys
+IS_PY2 = sys.version_info[0] == 2
+
+
+@cython.cclass
+class X(object):
+    x = cython.declare(cython.int)
+
+    def __init__(self, x):
+        self.x = x
+
+    def __repr__(self):
+        return "<%d>" % self.x
+
+
+@cython.cfunc
+@cython.locals(x=X)
+def x_of(x):
+    return x.x
+
+
+@cython.cclass
+class ClassEq(X):
+    """
+    >>> a = ClassEq(1)
+    >>> b = ClassEq(2)
+    >>> c = ClassEq(1)
+    >>> a == a
+    True
+    >>> a != a
+    False
+
+    >>> a == b
+    False
+    >>> a != b
+    True
+
+    >>> a == c
+    True
+    >>> if IS_PY2 and not compiled: a is c
+    ... else: a != c
+    False
+
+    >>> b == c
+    False
+    >>> b != c
+    True
+
+    >>> c == a
+    True
+    >>> if IS_PY2 and not compiled: c is a
+    ... else: c != a
+    False
+
+    >>> b == a
+    False
+    >>> b != a
+    True
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a < b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a > b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a <= b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a >= b
+    Traceback (most recent call last):
+    TypeError...
+
+    >>> print(a.__eq__.__doc__)
+    EQ
+    """
+    def __eq__(self, other):
+        """EQ"""
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassEq), type(self)
+        if isinstance(other, X):
+            return self.x == x_of(other)
+        elif isinstance(other, int):
+            return self.x < other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassEqNe(ClassEq):
+    """
+    >>> a = ClassEqNe(1)
+    >>> b = ClassEqNe(2)
+    >>> c = ClassEqNe(1)
+    >>> a == a
+    True
+    >>> a != a
+    False
+
+    >>> a == b
+    False
+    >>> a != b
+    True
+
+    >>> a == c
+    True
+    >>> a != c
+    False
+
+    >>> b == c
+    False
+    >>> b != c
+    True
+
+    >>> c == a
+    True
+    >>> c != a
+    False
+
+    >>> b == a
+    False
+    >>> b != a
+    True
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a < b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a > b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a <= b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a >= b
+    Traceback (most recent call last):
+    TypeError...
+
+    #>>> print(a.__eq__.__doc__)
+    #EQ
+    >>> print(a.__ne__.__doc__)
+    NE
+    """
+    def __ne__(self, other):
+        """NE"""
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassEqNe), type(self)
+        if isinstance(other, X):
+            return self.x != x_of(other)
+        elif isinstance(other, int):
+            return self.x < other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassEqNeGe(ClassEqNe):
+    """
+    >>> a = ClassEqNeGe(1)
+    >>> b = ClassEqNeGe(2)
+    >>> c = ClassEqNeGe(1)
+    >>> a == a
+    True
+    >>> a != a
+    False
+    >>> a >= a
+    True
+    >>> a <= a
+    True
+
+    >>> a == b
+    False
+    >>> a != b
+    True
+    >>> a >= b
+    False
+    >>> b <= a
+    False
+
+    >>> a == c
+    True
+    >>> a != c
+    False
+    >>> a >= c
+    True
+    >>> c <= a
+    True
+
+    >>> b == c
+    False
+    >>> b != c
+    True
+    >>> b >= c
+    True
+    >>> c <= b
+    True
+
+    >>> c == a
+    True
+    >>> c != a
+    False
+    >>> c >= a
+    True
+    >>> a <= c
+    True
+
+    >>> b == a
+    False
+    >>> b != a
+    True
+    >>> b >= a
+    True
+    >>> a <= b
+    True
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a < b
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a > b
+    Traceback (most recent call last):
+    TypeError...
+
+    >>> 2 <= a
+    False
+    >>> a >= 2
+    False
+    >>> 1 <= a
+    True
+    >>> a >= 1
+    True
+    >>> a >= 2
+    False
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 'x' <= a
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a >= 'x'
+    Traceback (most recent call last):
+    TypeError...
+
+    #>>> print(a.__eq__.__doc__)
+    #EQ
+    #>>> print(a.__ne__.__doc__)
+    #NE
+    >>> print(a.__ge__.__doc__)
+    GE
+   """
+    def __ge__(self, other):
+        """GE"""
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassEqNeGe), type(self)
+        if isinstance(other, X):
+            return self.x >= x_of(other)
+        elif isinstance(other, int):
+            return self.x >= other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassRichcmpOverride(ClassEqNeGe):
+    """
+    >>> a = ClassRichcmpOverride(1)
+    >>> b = ClassRichcmpOverride(1)
+
+    >>> a == a
+    True
+    >>> a != a
+    False
+
+    >>> a != b if compiled else a == b  # Python ignores __richcmp__()
+    True
+    >>> a == b if compiled else a != b  # Python ignores __richcmp__()
+    False
+
+    >>> if IS_PY2 or not compiled: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a >= b  # should no longer work when __richcmp__ is overwritten
+    Traceback (most recent call last):
+    TypeError...
+    """
+    def __richcmp__(self, other, op):
+        return NotImplemented
+
+
+@cython.cclass
+class ClassLe(X):
+    """
+    >>> a = ClassLe(1)
+    >>> b = ClassLe(2)
+    >>> c = ClassLe(1)
+
+    >>> a <= b
+    True
+    >>> b >= a
+    True
+    >>> b <= a
+    False
+    >>> a >= b
+    False
+
+    >>> a <= c
+    True
+    >>> c >= a
+    True
+    >>> c <= a
+    True
+    >>> a >= c
+    True
+
+    >>> b <= c
+    False
+    >>> c >= b
+    False
+    >>> c <= b
+    True
+    >>> b >= c
+    True
+
+    >>> 2 >= a
+    True
+    >>> a <= 2
+    True
+    >>> 1 >= a
+    True
+    >>> a <= 1
+    True
+    >>> a <= 0
+    False
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 'x' >= a
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a <= 'x'
+    Traceback (most recent call last):
+    TypeError...
+    """
+    def __le__(self, other):
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassLe), type(self)
+        if isinstance(other, X):
+            return self.x <= x_of(other)
+        elif isinstance(other, int):
+            return self.x <= other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassLt(X):
+    """
+    >>> a = ClassLt(1)
+    >>> b = ClassLt(2)
+    >>> c = ClassLt(1)
+
+    >>> a < b
+    True
+    >>> b > a
+    True
+    >>> b < a
+    False
+    >>> a > b
+    False
+
+    >>> a < c
+    False
+    >>> c > a
+    False
+    >>> c < a
+    False
+    >>> a > c
+    False
+
+    >>> b < c
+    False
+    >>> c > b
+    False
+    >>> c < b
+    True
+    >>> b > c
+    True
+
+    >>> sorted([a, b, c])
+    [<1>, <1>, <2>]
+    >>> sorted([b, a, c])
+    [<1>, <1>, <2>]
+
+    >>> 2 > a
+    True
+    >>> a < 2
+    True
+    >>> 1 > a
+    False
+    >>> a < 1
+    False
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 1 < a
+    Traceback (most recent call last):
+    TypeError...
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 'x' > a
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a < 'x'
+    Traceback (most recent call last):
+    TypeError...
+    """
+    def __lt__(self, other):
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassLt), type(self)
+        if isinstance(other, X):
+            return self.x < x_of(other)
+        elif isinstance(other, int):
+            return self.x < other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassLtGtInherited(X):
+    """
+    >>> a = ClassLtGtInherited(1)
+    >>> b = ClassLtGtInherited(2)
+    >>> c = ClassLtGtInherited(1)
+
+    >>> a < b
+    True
+    >>> b > a
+    True
+    >>> b < a
+    False
+    >>> a > b
+    False
+
+    >>> a < c
+    False
+    >>> c > a
+    False
+    >>> c < a
+    False
+    >>> a > c
+    False
+
+    >>> b < c
+    False
+    >>> c > b
+    False
+    >>> c < b
+    True
+    >>> b > c
+    True
+
+    >>> sorted([a, b, c])
+    [<1>, <1>, <2>]
+    >>> sorted([b, a, c])
+    [<1>, <1>, <2>]
+    """
+    def __gt__(self, other):
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassLtGtInherited), type(self)
+        if isinstance(other, X):
+            return self.x > x_of(other)
+        elif isinstance(other, int):
+            return self.x > other
+        return NotImplemented
+
+
+@cython.cclass
+class ClassLtGt(X):
+    """
+    >>> a = ClassLtGt(1)
+    >>> b = ClassLtGt(2)
+    >>> c = ClassLtGt(1)
+
+    >>> a < b
+    True
+    >>> b > a
+    True
+    >>> b < a
+    False
+    >>> a > b
+    False
+
+    >>> a < c
+    False
+    >>> c > a
+    False
+    >>> c < a
+    False
+    >>> a > c
+    False
+
+    >>> b < c
+    False
+    >>> c > b
+    False
+    >>> c < b
+    True
+    >>> b > c
+    True
+
+    >>> sorted([a, b, c])
+    [<1>, <1>, <2>]
+    >>> sorted([b, a, c])
+    [<1>, <1>, <2>]
+
+    >>> 2 > a
+    True
+    >>> 2 < a
+    False
+    >>> a < 2
+    True
+    >>> a > 2
+    False
+
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 'x' > a
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: 'x' < a
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a < 'x'
+    Traceback (most recent call last):
+    TypeError...
+    >>> if IS_PY2: raise TypeError  # doctest: +ELLIPSIS
+    ... else: a > 'x'
+    Traceback (most recent call last):
+    TypeError...
+    """
+    def __lt__(self, other):
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassLtGt), type(self)
+        if isinstance(other, X):
+            return self.x < x_of(other)
+        elif isinstance(other, int):
+            return self.x < other
+        return NotImplemented
+
+    def __gt__(self, other):
+        assert 1 <= self.x <= 2
+        assert isinstance(self, ClassLtGt), type(self)
+        if isinstance(other, X):
+            return self.x > x_of(other)
+        elif isinstance(other, int):
+            return self.x > other
+        return NotImplemented
+
+
+@cython.cclass
+class List(list):
+    """
+    >>> l = [1, 2, 3, 4]
+    >>> notl = List(l)
+    >>> notl == l
+    False
+    >>> notl != l     # implemented by base type
+    False
+    >>> notl == notl
+    True
+    >>> notl != notl  # implemented by base type
+    False
+    """
+    def __eq__(self, other):
+        return self is other or list(self) != list(other)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extended_unpacking_T235.pyx cython-0.20.1+1~202203241016-9537/tests/run/extended_unpacking_T235.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/extended_unpacking_T235.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extended_unpacking_T235.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 235
+# ticket: t235
 
 __doc__ = u"""
     >>> class FakeSeq(object):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extended_unpacking_T409.pyx cython-0.20.1+1~202203241016-9537/tests/run/extended_unpacking_T409.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/extended_unpacking_T409.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extended_unpacking_T409.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 409
+# ticket: t409
 
 def simple():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extern_builtins_T258.pyx cython-0.20.1+1~202203241016-9537/tests/run/extern_builtins_T258.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/extern_builtins_T258.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extern_builtins_T258.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 258
+# ticket: t258
 
 cdef extern from "Python.h":
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extern_impl_excvalue.srctree cython-0.20.1+1~202203241016-9537/tests/run/extern_impl_excvalue.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/extern_impl_excvalue.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extern_impl_excvalue.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,32 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import foo"
+PYTHON -c "import a"
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("*.pyx"),
+)
+
+######## foo.pxd ########
+
+cdef int bar() except *
+
+######## foo.pyx ########
+
+cdef extern from "bar_impl.c":
+    int bar() except *
+
+######## bar_impl.c ########
+
+static int bar() { return -1; }
+
+######## a.pyx ########
+
+cimport cython
+from foo cimport bar
+
+assert bar() == -1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extern_impl.srctree cython-0.20.1+1~202203241016-9537/tests/run/extern_impl.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/extern_impl.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extern_impl.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -13,12 +13,12 @@
 
 ######## foo.pxd ########
 
-cdef void bar()
+cdef void bar() except *
 
 ######## foo.pyx ########
 
 cdef extern from "bar_impl.c":
-    void bar()
+    void bar() except *
 
 ######## bar_impl.c ########
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extern_include_order.srctree cython-0.20.1+1~202203241016-9537/tests/run/extern_include_order.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/extern_include_order.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extern_include_order.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,56 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import a"
+PYTHON -c "import b"
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+    ext_modules = cythonize("*.pyx"),
+)
+
+######## a.pxd ########
+# cython: preliminary_late_includes_cy28=True
+
+cdef extern from "a_early.h":
+  ctypedef int my_int
+
+cdef extern from "a_late.h":
+    my_int square_value_plus_one()
+
+cdef my_int my_value "my_value"
+
+cdef my_int square "square"(my_int x)
+
+######## a.pyx ########
+
+my_value = 10
+
+cdef my_int square "square"(my_int x):
+    return x * x
+
+assert square_value_plus_one() == 101
+
+# Square must be explicitly used for its proto to be generated.
+cdef my_int use_square(x):
+  return square(x)
+
+######## a_early.h ########
+
+typedef int my_int;
+
+######## a_late.h ########
+
+static my_int square_value_plus_one() {
+  return square(my_value) + 1;
+}
+
+######## b.pyx ########
+
+cimport a
+
+# Likewise, a.square must be explicitly used.
+assert a.square(a.my_value) + 1 == 101
+assert a.square_value_plus_one() == 101
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ext_instance_type_T232.pyx cython-0.20.1+1~202203241016-9537/tests/run/ext_instance_type_T232.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ext_instance_type_T232.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ext_instance_type_T232.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 232
+# ticket: t232
 
 cdef class MyExt:
     cdef object attr
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extra_walrus.py cython-0.20.1+1~202203241016-9537/tests/run/extra_walrus.py
--- cython-0.20.1+1~201611251650-6686/tests/run/extra_walrus.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extra_walrus.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,146 @@
+# mode: run
+# tag: pure3.8
+
+# These are extra tests for the assignment expression/walrus operator/named expression that cover things
+# additional to the standard Python test-suite in tests/run/test_named_expressions.pyx
+
+import cython
+import sys
+
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+def optimized(x):
+    """
+    x*2 is optimized to a PythonCapiCallNode. The test fails unless the CloneNode is kept up-to-date
+    (in the event that the optimization changes and test_assert_path_exists fails, the thing to do
+    is to find another case that's similarly optimized - the test isn't specifically interested in
+    multiplication)
+
+    >>> optimized(5)
+    10
+    """
+    return (x:=x*2)
+
+# FIXME: currently broken; GH-4146
+# Changing x in the assignment expression should not affect the value used on the right-hand side
+#def order(x):
+#    """
+#    >>> order(5)
+#    15
+#    """
+#    return x+(x:=x*2)
+
+@cython.test_fail_if_path_exists("//CloneNode")
+def optimize_literals1():
+    """
+    There's a small optimization for literals to avoid creating unnecessary temps
+    >>> optimize_literals1()
+    10
+    """
+    x = 5
+    return (x := 10)
+
+@cython.test_fail_if_path_exists("//CloneNode")
+def optimize_literals2():
+    """
+    There's a small optimization for literals to avoid creating unnecessary temps
+    Test is in __doc__ (for Py2 string formatting reasons)
+    """
+    x = 5
+    return (x := u"a string")
+
+@cython.test_fail_if_path_exists("//CloneNode")
+def optimize_literals3():
+    """
+    There's a small optimization for literals to avoid creating unnecessary temps
+    Test is in __doc__ (for Py2 string formatting reasons)
+    """
+    x = 5
+    return (x := b"a bytes")
+
+@cython.test_fail_if_path_exists("//CloneNode")
+def optimize_literals4():
+    """
+    There's a small optimization for literals to avoid creating unnecessary temps
+    Test is in __doc__ (for Py2 string formatting reasons)
+    """
+    x = 5
+    return (x := (u"tuple", 1, 1.0, b"stuff"))
+
+if sys.version_info[0] != 2:
+    __doc__ = """
+        >>> optimize_literals2()
+        'a string'
+        >>> optimize_literals3()
+        b'a bytes'
+        >>> optimize_literals4()
+        ('tuple', 1, 1.0, b'stuff')
+        """
+else:
+    __doc__ = """
+        >>> optimize_literals2()
+        u'a string'
+        >>> optimize_literals3()
+        'a bytes'
+        >>> optimize_literals4()
+        (u'tuple', 1, 1.0, 'stuff')
+        """
+
+
+@cython.test_fail_if_path_exists("//CoerceToPyTypeNode//AssignmentExpressionNode")
+def avoid_extra_coercion(x : cython.double):
+    """
+    The assignment expression and x are both coerced to PyObject - this should happen only once
+    rather than to both separately
+    >>> avoid_extra_coercion(5.)
+    5.0
+    """
+    y : object = "I'm an object"
+    return (y := x)
+
+async def async_func():
+    """
+    DW doesn't understand async functions well enough to make it a runtime test, but it was causing
+    a compile-time failure at one point
+    """
+    if variable := 1:
+        pass
+
+y_global = 6
+
+class InLambdaInClass:
+    """
+    >>> InLambdaInClass.x1
+    12
+    >>> InLambdaInClass.x2
+    [12, 12]
+    """
+    x1 = (lambda y_global: (y_global := y_global + 1) + y_global)(2) + y_global
+    x2 = [(lambda y_global: (y_global := y_global + 1) + y_global)(2) + y_global for _ in range(2) ]
+
+def in_lambda_in_list_comprehension1():
+    """
+    >>> in_lambda_in_list_comprehension1()
+    [[0, 2, 4, 6], [0, 2, 4, 6], [0, 2, 4, 6], [0, 2, 4, 6], [0, 2, 4, 6]]
+    """
+    return [ (lambda x: [(x := y) + x for y in range(4)])(x) for x in range(5) ]
+
+def in_lambda_in_list_comprehension2():
+    """
+    >>> in_lambda_in_list_comprehension2()
+    [[0, 1, 2, 3], [1, 2, 3, 4], [2, 3, 4, 5], [3, 4, 5, 6], [4, 5, 6, 7]]
+    """
+    return [ (lambda z: [(x := y) + z for y in range(4)])(x) for x in range(5) ]
+
+def in_lambda_in_generator_expression1():
+    """
+    >>> in_lambda_in_generator_expression1()
+    [(0, 2, 4, 6), (0, 2, 4, 6), (0, 2, 4, 6), (0, 2, 4, 6), (0, 2, 4, 6)]
+    """
+    return [ (lambda x: tuple((x := y) + x for y in range(4)))(x) for x in range(5) ]
+
+def in_lambda_in_generator_expression2():
+    """
+    >>> in_lambda_in_generator_expression2()
+    [(0, 1, 2, 3), (1, 2, 3, 4), (2, 3, 4, 5), (3, 4, 5, 6), (4, 5, 6, 7)]
+    """
+    return [ (lambda z: tuple((x := y) + z for y in range(4)))(x) for x in range(5) ]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/extstarargs.pyx cython-0.20.1+1~202203241016-9537/tests/run/extstarargs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/extstarargs.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/extstarargs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,124 +1,169 @@
-__doc__ = u"""
-    >>> s = Silly(1,2,3, 'test')
-    >>> (spam,grail,swallow,creosote,onlyt,onlyk,tk) = (
-    ...     s.spam,s.grail,s.swallow,s.creosote,s.onlyt,s.onlyk,s.tk)
-
-    >>> spam(1,2,3)
-    (1, 2, 3)
-    >>> spam(1,2)
-    Traceback (most recent call last):
-    TypeError: spam() takes exactly 3 positional arguments (2 given)
-    >>> spam(1,2,3,4)
-    Traceback (most recent call last):
-    TypeError: spam() takes exactly 3 positional arguments (4 given)
-    >>> spam(1,2,3, a=1) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-    TypeError: spam() got an unexpected keyword argument 'a'
-
-    >>> grail(1,2,3)
-    (1, 2, 3, ())
-    >>> grail(1,2,3,4)
-    (1, 2, 3, (4,))
-    >>> grail(1,2,3,4,5,6,7,8,9)
-    (1, 2, 3, (4, 5, 6, 7, 8, 9))
-    >>> grail(1,2)
-    Traceback (most recent call last):
-    TypeError: grail() takes at least 3 positional arguments (2 given)
-    >>> grail(1,2,3, a=1) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-    TypeError: grail() got an unexpected keyword argument 'a'
-
-    >>> swallow(1,2,3)
-    (1, 2, 3, ())
-    >>> swallow(1,2,3,4)
-    Traceback (most recent call last):
-    TypeError: swallow() takes exactly 3 positional arguments (4 given)
-    >>> swallow(1,2,3, a=1, b=2)
-    (1, 2, 3, (('a', 1), ('b', 2)))
-    >>> swallow(1,2,3, x=1)
-    Traceback (most recent call last):
-    TypeError: swallow() got multiple values for keyword argument 'x'
-
-    >>> creosote(1,2,3)
-    (1, 2, 3, (), ())
-    >>> creosote(1,2,3,4)
-    (1, 2, 3, (4,), ())
-    >>> creosote(1,2,3, a=1)
-    (1, 2, 3, (), (('a', 1),))
-    >>> creosote(1,2,3,4, a=1, b=2)
-    (1, 2, 3, (4,), (('a', 1), ('b', 2)))
-    >>> creosote(1,2,3,4, x=1)
-    Traceback (most recent call last):
-    TypeError: creosote() got multiple values for keyword argument 'x'
-
-    >>> onlyt(1)
-    (1,)
-    >>> onlyt(1,2)
-    (1, 2)
-    >>> onlyt(a=1)
-    Traceback (most recent call last):
-    TypeError: onlyt() got an unexpected keyword argument 'a'
-    >>> onlyt(1, a=2)
-    Traceback (most recent call last):
-    TypeError: onlyt() got an unexpected keyword argument 'a'
-
-    >>> onlyk(a=1)
-    (('a', 1),)
-    >>> onlyk(a=1, b=2)
-    (('a', 1), ('b', 2))
-    >>> onlyk(1)
-    Traceback (most recent call last):
-    TypeError: onlyk() takes exactly 0 positional arguments (1 given)
-    >>> onlyk(1, 2)
-    Traceback (most recent call last):
-    TypeError: onlyk() takes exactly 0 positional arguments (2 given)
-    >>> onlyk(1, a=1, b=2)
-    Traceback (most recent call last):
-    TypeError: onlyk() takes exactly 0 positional arguments (1 given)
-
-    >>> tk(a=1)
-    (('a', 1),)
-    >>> tk(a=1, b=2)
-    (('a', 1), ('b', 2))
-    >>> tk(1)
-    (1,)
-    >>> tk(1, 2)
-    (1, 2)
-    >>> tk(1, a=1, b=2)
-    (1, ('a', 1), ('b', 2))
-"""
-
-import sys, re
-if sys.version_info >= (2,6):
-    __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__, re.M)
+cimport cython
 
 cdef sorteditems(d):
-    l = list(d.items())
-    l.sort()
-    return tuple(l)
+    return tuple(sorted(d.items()))
+
 
 cdef class Silly:
 
     def __init__(self, *a):
-        pass
+        """
+        >>> s = Silly(1,2,3, 'test')
+        """
 
     def spam(self, x, y, z):
+        """
+        >>> s = Silly()
+        >>> s.spam(1,2,3)
+        (1, 2, 3)
+        >>> s.spam(1,2)
+        Traceback (most recent call last):
+        TypeError: spam() takes exactly 3 positional arguments (2 given)
+        >>> s.spam(1,2,3,4)
+        Traceback (most recent call last):
+        TypeError: spam() takes exactly 3 positional arguments (4 given)
+        >>> s.spam(1,2,3, a=1)
+        Traceback (most recent call last):
+        TypeError: spam() got an unexpected keyword argument 'a'
+        """
         return (x, y, z)
 
     def grail(self, x, y, z, *a):
+        """
+        >>> s = Silly()
+        >>> s.grail(1,2,3)
+        (1, 2, 3, ())
+        >>> s.grail(1,2,3,4)
+        (1, 2, 3, (4,))
+        >>> s.grail(1,2,3,4,5,6,7,8,9)
+        (1, 2, 3, (4, 5, 6, 7, 8, 9))
+        >>> s.grail(1,2)
+        Traceback (most recent call last):
+        TypeError: grail() takes at least 3 positional arguments (2 given)
+        >>> s.grail(1,2,3, a=1)
+        Traceback (most recent call last):
+        TypeError: grail() got an unexpected keyword argument 'a'
+        """
         return (x, y, z, a)
 
     def swallow(self, x, y, z, **k):
+        """
+        >>> s = Silly()
+        >>> s.swallow(1,2,3)
+        (1, 2, 3, ())
+        >>> s.swallow(1,2,3,4)
+        Traceback (most recent call last):
+        TypeError: swallow() takes exactly 3 positional arguments (4 given)
+        >>> s.swallow(1,2,3, a=1, b=2)
+        (1, 2, 3, (('a', 1), ('b', 2)))
+        >>> s.swallow(1,2,3, x=1)
+        Traceback (most recent call last):
+        TypeError: swallow() got multiple values for keyword argument 'x'
+        """
         return (x, y, z, sorteditems(k))
 
     def creosote(self, x, y, z, *a, **k):
+        """
+        >>> s = Silly()
+        >>> s.creosote(1,2,3)
+        (1, 2, 3, (), ())
+        >>> s.creosote(1,2,3,4)
+        (1, 2, 3, (4,), ())
+        >>> s.creosote(1,2,3, a=1)
+        (1, 2, 3, (), (('a', 1),))
+        >>> s.creosote(1,2,3,4, a=1, b=2)
+        (1, 2, 3, (4,), (('a', 1), ('b', 2)))
+        >>> s.creosote(1,2,3,4, x=1)
+        Traceback (most recent call last):
+        TypeError: creosote() got multiple values for keyword argument 'x'
+        """
         return (x, y, z, a, sorteditems(k))
 
     def onlyt(self, *a):
+        """
+        >>> s = Silly()
+        >>> s.onlyt(1)
+        (1,)
+        >>> s.onlyt(1,2)
+        (1, 2)
+        >>> s.onlyt(a=1)
+        Traceback (most recent call last):
+        TypeError: onlyt() got an unexpected keyword argument 'a'
+        >>> s.onlyt(1, a=2)
+        Traceback (most recent call last):
+        TypeError: onlyt() got an unexpected keyword argument 'a'
+        """
+        return a
+
+    @cython.binding(False)  # passthrough of exact same tuple can't work with binding
+    def onlyt_nobinding(self, *a):
+        """
+        >>> s = Silly()
+        >>> s.onlyt_nobinding(1)
+        (1,)
+        >>> s.onlyt_nobinding(1,2)
+        (1, 2)
+        >>> s.onlyt_nobinding(a=1)
+        Traceback (most recent call last):
+        TypeError: onlyt_nobinding() got an unexpected keyword argument 'a'
+        >>> s.onlyt_nobinding(1, a=2)
+        Traceback (most recent call last):
+        TypeError: onlyt_nobinding() got an unexpected keyword argument 'a'
+        >>> test_no_copy_args(s.onlyt_nobinding)
+        True
+        """
         return a
 
     def onlyk(self, **k):
+        """
+        >>> s = Silly()
+        >>> s.onlyk(a=1)
+        (('a', 1),)
+        >>> s.onlyk(a=1, b=2)
+        (('a', 1), ('b', 2))
+        >>> s.onlyk(1)
+        Traceback (most recent call last):
+        TypeError: onlyk() takes exactly 0 positional arguments (1 given)
+        >>> s.onlyk(1, 2)
+        Traceback (most recent call last):
+        TypeError: onlyk() takes exactly 0 positional arguments (2 given)
+        >>> s.onlyk(1, a=1, b=2)
+        Traceback (most recent call last):
+        TypeError: onlyk() takes exactly 0 positional arguments (1 given)
+        """
         return sorteditems(k)
 
     def tk(self, *a, **k):
+        """
+        >>> s = Silly()
+        >>> s.tk(a=1)
+        (('a', 1),)
+        >>> s.tk(a=1, b=2)
+        (('a', 1), ('b', 2))
+        >>> s.tk(1)
+        (1,)
+        >>> s.tk(1, 2)
+        (1, 2)
+        >>> s.tk(1, a=1, b=2)
+        (1, ('a', 1), ('b', 2))
+        """
         return a + sorteditems(k)
+
+    @cython.binding(False)  # passthrough of exact same tuple can't work with binding
+    def t_kwonly(self, *a, k):
+        """
+        >>> s = Silly()
+        >>> test_no_copy_args(s.t_kwonly, k=None)
+        True
+        """
+        return a
+
+
+def test_no_copy_args(func, **kw):
+    """
+    func is a function such that func(*args, **kw) returns args.
+    We test that no copy is made of the args tuple.
+    This tests both the caller side and the callee side.
+    """
+    args = (1, 2, 3)
+    return func(*args, **kw) is args
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exttype_freelist.pyx cython-0.20.1+1~202203241016-9537/tests/run/exttype_freelist.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exttype_freelist.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exttype_freelist.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -451,3 +451,19 @@
 
     def __init__(self, obj=None):
         self.attribute = obj
+
+
+@cython.freelist(3)
+@cython.cclass
+class DecoratedPyClass(object):
+    """
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ext_type_none_arg.pyx cython-0.20.1+1~202203241016-9537/tests/run/ext_type_none_arg.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ext_type_none_arg.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ext_type_none_arg.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,10 @@
 
 cimport cython
+try:
+    import typing
+    from typing import Optional
+except ImportError:
+    pass  # Cython can still identify the use of "typing" even if the module doesn't exist
 
 
 ### extension types
@@ -79,6 +84,39 @@
     """
     return attr(x)
 
+def ext_annotations(x: MyExtType):
+    """
+    Behaves the same as "MyExtType x not None"
+    >>> ext_annotations(MyExtType())
+    123
+    >>> ext_annotations(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'x' has incorrect type (expected ext_type_none_arg.MyExtType, got NoneType)
+    """
+    return attr(x)
+
+@cython.allow_none_for_extension_args(False)
+def ext_annotations_check_on(x: MyExtType):
+    """
+    >>> ext_annotations_check_on(MyExtType())
+    123
+    >>> ext_annotations_check_on(None)
+    Traceback (most recent call last):
+    TypeError: Argument 'x' has incorrect type (expected ext_type_none_arg.MyExtType, got NoneType)
+    """
+    return attr(x)
+
+def ext_optional(x: typing.Optional[MyExtType], y: Optional[MyExtType]):
+    """
+    Behaves the same as "or None"
+    >>> ext_optional(MyExtType(), MyExtType())
+    246
+    >>> ext_optional(MyExtType(), None)
+    444
+    >>> ext_optional(None, MyExtType())
+    444
+    """
+    return attr(x) + attr(y)
 
 ### builtin types (using list)
 
@@ -166,6 +204,30 @@
     'NoneType'
     """
     return type(o).__name__
+
+@cython.allow_none_for_extension_args(False)
+def object_default_annotation(o : object):
+    """
+    >>> object_default_annotation(object())
+    'object'
+    >>> object_default_annotation([])
+    'list'
+    >>> object_default_annotation(None)
+    'NoneType'
+    """
+    return type(o).__name__
+
+# no decorator
+def object_default_annotation2(o : object):
+    """
+    >>> object_default_annotation2(object())
+    'object'
+    >>> object_default_annotation2([])
+    'list'
+    >>> object_default_annotation2(None)
+    'NoneType'
+    """
+    return type(o).__name__
 
 @cython.allow_none_for_extension_args(False)
 def object_default_none(object o=None): # behaves like 'or None'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exttype.pyx cython-0.20.1+1~202203241016-9537/tests/run/exttype.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exttype.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exttype.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,52 @@
+# mode: run
+# tag: exttype, tpnew
+
+from __future__ import print_function
+
+from cpython.object cimport PyTypeObject
+
 
 cdef gobble(a, b):
-    print a, b
+    print(a, b)
+
+
+def tp_new_ptr(exttype):
+    assert isinstance(exttype, type)
+    tp =  exttype
+    return tp.tp_new
+
+
+cdef class Empty:
+    """
+    >>> n = Empty()
+    >>> isinstance(n, Empty)
+    True
+    >>> tp_new_ptr(Empty) != 0
+    True
+    """
+
+
+cdef class EmptySubclass(Empty):
+    """
+    >>> n = EmptySubclass()
+    >>> isinstance(n, EmptySubclass)
+    True
+    >>> tp_new_ptr(EmptySubclass) != 0
+    True
+    >>> tp_new_ptr(EmptySubclass) == tp_new_ptr(Empty)
+    True
+    """
+
+
+cdef class CInit:
+    """
+    >>> c = CInit()
+    >>> isinstance(c, CInit)
+    True
+    """
+    def __cinit__(self):
+        assert self is not None
+
 
 cdef class Spam:
     """
@@ -21,6 +67,7 @@
     def eat(self):
         gobble(self.eggs, self.ham)
 
+
 def f(Spam spam):
     """
     >>> s = Spam(12)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/exttype_total_ordering.pyx cython-0.20.1+1~202203241016-9537/tests/run/exttype_total_ordering.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/exttype_total_ordering.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/exttype_total_ordering.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,1020 @@
+# mode: run
+# tag: total_ordering
+
+from __future__ import print_function
+
+"""
+    >>> class PyTotalOrdering:
+    ...     def __init__(self, value):
+    ...         self.value = value
+    ...     def __eq__(self, other):
+    ...         return self.value == other.value
+    ...     def __lt__(self, other):
+    ...         return self.value < other.value
+    >>> test_all_comp(functools.total_ordering(PyTotalOrdering))
+    True
+"""
+
+cimport cython
+import functools
+import operator
+
+COMPARISONS = [
+    # Don't test equals, the directive doesn't add that.
+    # ('==', operator.__eq__),
+    ('!=', operator.__ne__),
+    ('<',  operator.__lt__),
+    ('>',  operator.__gt__),
+    ('<=', operator.__le__),
+    ('>=', operator.__ge__),
+]
+
+def test_all_comp(cls):
+    """Check every combination of comparison operators."""
+    a, b, c = 10, 15, 20
+    succeeded = True
+    for comp, func in COMPARISONS:
+        for left in [cls(a), cls(b), cls(c)]:
+            for right in [ValueHolder(a), ValueHolder(b), ValueHolder(c)]:
+                expected = func(left.value, right.value)
+                try:
+                    result = func(left, right)
+                    # repeat to rule out deallocation bugs (and assert determinism)
+                    for _ in range(10):
+                        assert result == func(left, right)
+                except TypeError:
+                    print("TypeError:", left.value, comp, right.value)
+                    succeeded = False
+                else:
+                    if expected != result:
+                        print(
+                            left.value, comp, right.value,
+                            "expected:", expected, "got:", result
+                        )
+                        succeeded = False
+    return succeeded
+
+class ValueHolder:
+    """Has a value, but can't compare."""
+    def __init__(self, value):
+        self.value = value
+
+
+
+cdef class ExtTypeNoTotalOrdering:
+    """
+    >>> a = ExtTypeNoTotalOrdering(5)
+    >>> b = ExtTypeNoTotalOrdering(10)
+    >>> a == b
+    False
+    >>> a != b  # Added in Python 3, but Cython backports
+    True
+    >>> a < b
+    True
+    >>> b < a
+    False
+    >>> a > b
+    False
+    >>> b > a
+    True
+    >>> import sys
+    >>> try: _ =  a >= b
+    ... except TypeError:
+    ...     assert sys.version_info[0] >= 3
+    ... else:
+    ...     assert sys.version_info[0] < 3
+    >>> try: _ =  a <= b
+    ... except TypeError:
+    ...     assert sys.version_info[0] >= 3
+    ... else:
+    ...     assert sys.version_info[0] < 3
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+# Every combination of methods which is valid.
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingNeLtGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingNeLtGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqLtGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqLtGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtGt:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGt)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtGtGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtGtLe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtLe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+@cython.total_ordering
+cdef class ExtTypeTotalOrderingEqNeLtGtLeGe:
+    """
+    >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtLeGe)
+    True
+    """
+    cdef public int value
+    def __init__(self, val):
+        self.value = val
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+    def __ne__(self, other):
+        return self.value != other.value
+
+    def __lt__(self, other):
+        return self.value < other.value
+
+    def __gt__(self, other):
+        return self.value > other.value
+
+    def __le__(self, other):
+        return self.value <= other.value
+
+    def __ge__(self, other):
+        return self.value >= other.value
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fastcall.pyx cython-0.20.1+1~202203241016-9537/tests/run/fastcall.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fastcall.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fastcall.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,128 @@
+# mode: run
+# tag: METH_FASTCALL
+
+cimport cython
+
+import sys
+import struct
+from collections import deque
+
+pack = struct.pack
+
+
+def deque_methods(v):
+    """
+    >>> deque_methods(2)
+    [1, 2, 3, 4]
+    """
+    d = deque([1, 3, 4])
+    assert list(d) == [1,3,4]
+    if sys.version_info >= (3, 5):
+        d.insert(1, v)
+    else:
+        # deque has no 2-args methods in older Python versions
+        d.rotate(-1)
+        d.appendleft(2)
+        d.rotate(1)
+    assert list(d) == [1,2,3,4]
+    d.rotate(len(d) // 2)
+    assert list(d) == [3,4,1,2]
+    d.rotate(len(d) // 2)
+    assert list(d) == [1,2,3,4]
+
+    return list(d)
+
+
+def struct_methods(v):
+    """
+    >>> i, lf, i2, f = struct_methods(2)
+    >>> struct.unpack('i', i)
+    (2,)
+    >>> struct.unpack('i', i2)
+    (2,)
+    >>> struct.unpack('lf', lf)
+    (2, 4.0)
+    >>> struct.unpack('f', f)
+    (2.0,)
+    """
+    local_pack = pack
+    return [
+        struct.pack('i', v),
+        struct.pack('lf', v, v*2),
+        pack('i', v),
+        local_pack('f', v),
+    ]
+
+
+cdef class SelfCast:
+    """
+    >>> f = SelfCast()
+    >>> f.index_of_self([f])
+    0
+    >>> f.index_of_self([])  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ValueError...
+    """
+    def index_of_self(self, list orbit not None):
+        return orbit.index(self)
+
+
+cdef extern from *:
+    int PyCFunction_GET_FLAGS(op)
+
+
+def has_fastcall(meth):
+    """
+    Given a builtin_function_or_method or cyfunction ``meth``,
+    return whether it uses ``METH_FASTCALL``.
+    """
+    # Hardcode METH_FASTCALL constant equal to 0x80 for simplicity
+    return bool(PyCFunction_GET_FLAGS(meth) & 0x80)
+
+
+def assert_fastcall(meth):
+    """
+    Assert that ``meth`` uses ``METH_FASTCALL`` if the Python
+    implementation supports it.
+    """
+    # getattr uses METH_FASTCALL on CPython >= 3.7
+    if has_fastcall(getattr) and not has_fastcall(meth):
+        raise AssertionError(f"{meth} does not use METH_FASTCALL")
+
+
+@cython.binding(False)
+def fastcall_function(**kw):
+    """
+    >>> assert_fastcall(fastcall_function)
+    """
+    return kw
+
+@cython.binding(True)
+def fastcall_cyfunction(**kw):
+    """
+    >>> assert_fastcall(fastcall_cyfunction)
+    """
+    return kw
+
+cdef class Dummy:
+    @cython.binding(False)
+    def fastcall_method(self, x, *args, **kw):
+        """
+        >>> assert_fastcall(Dummy().fastcall_method)
+        """
+        return tuple(args) + tuple(kw)
+
+cdef class CyDummy:
+    @cython.binding(True)
+    def fastcall_method(self, x, *args, **kw):
+        """
+        >>> assert_fastcall(CyDummy.fastcall_method)
+        """
+        return tuple(args) + tuple(kw)
+
+class PyDummy:
+    def fastcall_method(self, x, *args, **kw):
+        """
+        >>> assert_fastcall(PyDummy.fastcall_method)
+        """
+        return tuple(args) + tuple(kw)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/file_encoding_T740.py cython-0.20.1+1~202203241016-9537/tests/run/file_encoding_T740.py
--- cython-0.20.1+1~201611251650-6686/tests/run/file_encoding_T740.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/file_encoding_T740.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # encoding: koi8-r
 # mode: run
-# ticket: 740
+# ticket: t740
 """
 >>> wtf
 'wtf'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/final_method_T586.pyx cython-0.20.1+1~202203241016-9537/tests/run/final_method_T586.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/final_method_T586.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/final_method_T586.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 568
+# ticket: t568
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/float_division.pyx cython-0.20.1+1~202203241016-9537/tests/run/float_division.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/float_division.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/float_division.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -26,6 +26,39 @@
     return 3.0 / 2.0
 
 
+def div_by_0(x):
+    """
+    >>> div_by_0(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(1.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> float('inf') / 0.0  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(float('inf'))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(float('-inf'))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> float('nan') / 0.0  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_by_0(float('nan'))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    """
+    return x / 0.0
+
+
 def div_1_by(x):
     """
     >>> div_1_by(1.0)
@@ -42,6 +75,12 @@
     -0.0
     >>> div_1_by(float('nan'))
     nan
+    >>> div_1_by(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_1_by(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
     """
     return 1.0 / x
 
@@ -116,6 +155,12 @@
     nan
     >>> div_neg_2_by(float('nan'))
     nan
+    >>> div_neg_2_by(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_neg_2_by(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
     """
     return (-2.0) / x
 
@@ -148,6 +193,12 @@
     nan
     >>> div_nan_by(float('nan'))
     nan
+    >>> div_nan_by(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_nan_by(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
     """
     return float("nan") / x
 
@@ -182,6 +233,15 @@
     nan
     >>> div_inf_by(float('-inf'))
     nan
+    >>> float("inf") / 0.0  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_inf_by(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_inf_by(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
     """
     return float("inf") / x
 
@@ -196,5 +256,14 @@
     inf
     >>> div_neg_inf_by(-1.0)
     inf
+    >>> float("-inf") / 0.0  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_neg_inf_by(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
+    >>> div_neg_inf_by(0.0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: float division...
     """
     return float("-inf") / x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/float_floor_division_T260.pyx cython-0.20.1+1~202203241016-9537/tests/run/float_floor_division_T260.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/float_floor_division_T260.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/float_floor_division_T260.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 260
+# ticket: t260
 
 def floor_div_float(double a, double b):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/float_len_T480.pyx cython-0.20.1+1~202203241016-9537/tests/run/float_len_T480.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/float_len_T480.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/float_len_T480.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 480
+# ticket: t480
 
 def f(x):
     return x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_from_float_T254.pyx cython-0.20.1+1~202203241016-9537/tests/run/for_from_float_T254.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/for_from_float_T254.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_from_float_T254.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 254
+# ticket: t254
 
 def double_target(a, b):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_from_pyvar_loop_T601_extern_def.h cython-0.20.1+1~202203241016-9537/tests/run/for_from_pyvar_loop_T601_extern_def.h
--- cython-0.20.1+1~201611251650-6686/tests/run/for_from_pyvar_loop_T601_extern_def.h	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_from_pyvar_loop_T601_extern_def.h	1970-01-01 00:00:00.000000000 +0000
@@ -1,2 +0,0 @@
-
-typedef unsigned long Ulong;
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_from_pyvar_loop_T601.pyx cython-0.20.1+1~202203241016-9537/tests/run/for_from_pyvar_loop_T601.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/for_from_pyvar_loop_T601.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_from_pyvar_loop_T601.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 601
+# ticket: t601
 
 cdef unsigned long size2():
     return 3
@@ -26,7 +26,8 @@
         print j
 
 
-cdef extern from "for_from_pyvar_loop_T601_extern_def.h":
+cdef extern from *:
+    """typedef unsigned long Ulong;"""
     ctypedef unsigned long Ulong
 
 cdef Ulong size():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_in_break_continue_T533.pyx cython-0.20.1+1~202203241016-9537/tests/run/for_in_break_continue_T533.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/for_in_break_continue_T533.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_in_break_continue_T533.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 533
+# ticket: t533
 
 def for_in():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_in_iter.py cython-0.20.1+1~202203241016-9537/tests/run/for_in_iter.py
--- cython-0.20.1+1~201611251650-6686/tests/run/for_in_iter.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_in_iter.py	2022-03-24 10:16:46.000000000 +0000
@@ -61,6 +61,37 @@
         l.append(i)
     return l
 
+
+def listcomp_over_multiplied_constant_tuple():
+    """
+    >>> listcomp_over_multiplied_constant_tuple()
+    [[], [1, 2, 3], [1, 2, 3, 1, 2, 3], [1, 2, 3, 1, 2, 3, 1, 2, 3], [1, 2, 3, 1, 2, 3]]
+    """
+    return [
+        [i for i in (1, 2, 3) * 0],
+        [i for i in (1, 2, 3) * 1],
+        [i for i in (1, 2, 3) * 2],
+        [i for i in (1, 2, 3) * 3],
+        [i for i in (1, 2, 3) * 2],
+    ]
+
+
+@cython.test_assert_path_exists('//ReturnStatNode//ForInStatNode//TupleNode')
+@cython.test_fail_if_path_exists('//ReturnStatNode//ForInStatNode//ListNode')
+def listcomp_over_multiplied_constant_list():
+    """
+    >>> listcomp_over_multiplied_constant_list()
+    [[], [1, 2, 3], [1, 2, 3, 1, 2, 3], [1, 2, 3, 1, 2, 3, 1, 2, 3], [1, 2, 3, 1, 2, 3]]
+    """
+    return [
+        [i for i in [1, 2, 3] * 0],
+        [i for i in [1, 2, 3] * 1],
+        [i for i in [1, 2, 3] * 2],
+        [i for i in [1, 2, 3] * 3],
+        [i for i in [1, 2, 3] * 2],
+    ]
+
+
 class Iterable(object):
     """
     >>> for_in_pyiter(Iterable(5))
@@ -131,3 +162,12 @@
     """
     for i in range(N):
         yield i
+
+def for_in_range_invalid_arg_count():
+    """
+    >>> for_in_range_invalid_arg_count()     # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    """
+    for i in range(1, 2, 3, 4):
+        pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/for_in_range_T372.pyx cython-0.20.1+1~202203241016-9537/tests/run/for_in_range_T372.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/for_in_range_T372.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/for_in_range_T372.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,5 @@
-# ticket: 372
+# mode: run
+# ticket: t372
 
 cimport cython
 
@@ -22,6 +23,7 @@
     print
     return i,n
 
+
 @cython.test_assert_path_exists("//ForFromStatNode")
 @cython.test_fail_if_path_exists("//ForInStatNode")
 def test_negindex():
@@ -40,6 +42,7 @@
         n = 0
     return i,n
 
+
 @cython.test_assert_path_exists("//ForFromStatNode",
                                 "//ForFromStatNode//PrintStatNode//CoerceToPyTypeNode")
 @cython.test_fail_if_path_exists("//ForInStatNode")
@@ -58,6 +61,7 @@
         n = 0
     return i,n
 
+
 @cython.test_assert_path_exists("//ForFromStatNode")
 @cython.test_fail_if_path_exists("//ForInStatNode")
 def test_fix():
@@ -77,6 +81,7 @@
     print
     return i
 
+
 @cython.test_assert_path_exists("//ForFromStatNode")
 @cython.test_fail_if_path_exists("//ForInStatNode")
 def test_break():
@@ -99,6 +104,7 @@
     print
     return i,n
 
+
 @cython.test_assert_path_exists("//ForFromStatNode")
 @cython.test_fail_if_path_exists("//ForInStatNode")
 def test_return():
@@ -117,3 +123,24 @@
             return i,n
     print
     return "FAILED!"
+
+
+ctypedef enum RangeEnum:
+    EnumValue1
+    EnumValue2
+    EnumValue3
+
+
+@cython.test_assert_path_exists("//ForFromStatNode")
+@cython.test_fail_if_path_exists("//ForInStatNode")
+def test_enum_range():
+    """
+    # NOTE: it's not entirely clear that this is the expected behaviour, but that's how it currently is.
+    >>> test_enum_range()
+    'RangeEnum'
+    """
+    cdef RangeEnum n = EnumValue3
+    for i in range(n):
+        assert 0 <= i < n
+        assert cython.typeof(i) == "RangeEnum", cython.typeof(i)
+    return cython.typeof(i)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fstring.pyx cython-0.20.1+1~202203241016-9537/tests/run/fstring.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fstring.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fstring.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,23 +1,116 @@
 # mode: run
-# tag: f_strings, pep498
+# tag: f_strings, pep498, werror
 
 ####
 # Cython specific PEP 498 tests in addition to test_fstring.pyx from CPython
 ####
 
+cimport cython
+
 import sys
 IS_PYPY = hasattr(sys, 'pypy_version_info')
 
-cdef extern from *:
-    int INT_MAX
-    long LONG_MAX
-    long LONG_MIN
+from libc.limits cimport INT_MAX, LONG_MAX, LONG_MIN
 
 max_int = INT_MAX
 max_long = LONG_MAX
 min_long = LONG_MIN
 
 
+@cython.test_fail_if_path_exists(
+    "//JoinedStrNode",
+)
+@cython.test_assert_path_exists(
+    "//AddNode",
+)
+def concat_strings(a, b):
+    """
+    >>> concat_strings("", "")
+    x
+    
+    x
+    x
+    x
+    xx
+    >>> concat_strings("a", "")
+    ax
+    a
+    x
+    ax
+    ax
+    axx
+    >>> concat_strings("", "b")
+    x
+    b
+    xb
+    xb
+    xb
+    xxb
+    >>> concat_strings("a", "b")
+    ax
+    ab
+    xb
+    axb
+    axb
+    axxb
+    >>> concat_strings("".join(["a", "b"]), "")  # fresh temp string left
+    abx
+    ab
+    x
+    abx
+    abx
+    abxx
+    >>> concat_strings("", "".join(["a", "b"]))  # fresh temp string right
+    x
+    ab
+    xab
+    xab
+    xab
+    xxab
+    """
+    print(f"{a}x")
+    print(f"{a}{b}")
+    print(f"x{b}")
+    print(f"{a+'x'}{b}")      # fresh temp string left
+    print(f"{a}{'x'+b}")      # fresh temp string right
+    print(f"{a+'x'}{'x'+b}")  # fresh temp strings right and left
+
+
+@cython.test_fail_if_path_exists(
+    "//FormattedValueNode",
+    "//JoinedStrNode",
+    "//AddNode",
+)
+def escaping():
+    """
+    >>> escaping()
+    """
+    assert f'{{{{{"abc"}}}}}{{}}{{' == '{{abc}}{}{'
+    s = f'{{{{{"abc"}}}}}{{}}{{'
+    assert s == '{{abc}}{}{', s
+
+    assert f'\x7b}}' == '{}'
+    s = f'\x7b}}'
+    assert s == '{}', s
+
+    assert f'{"{{}}"}' == '{{}}'
+    s = f'{"{{}}"}'
+    assert s == '{{}}', s
+
+
+@cython.test_fail_if_path_exists(
+    "//FormattedValueNode",
+    "//JoinedStrNode",
+    "//AddNode",
+)
+def nested_constant():
+    """
+    >>> print(nested_constant())
+    xyabc123321
+    """
+    return f"""{f'''xy{f"abc{123}{'321'}"!s}'''}"""
+
+
 def format2(ab, cd):
     """
     >>> a, b, c = format2(1, 2)
@@ -45,6 +138,23 @@
     return a, b, c
 
 
+ctypedef enum TestValues:
+    enum_ABC = 1
+    enum_XYZ = 2
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
+def format_c_enum():
+    """
+    >>> s = format_c_enum()
+    >>> s == '1-2' or s
+    True
+    """
+    return f"{enum_ABC}-{enum_XYZ}"
+
+
 def format_c_numbers(signed char c, short s, int n, long l, float f, double d):
     """
     >>> s1, s2, s3, s4 = format_c_numbers(123, 135, 12, 12312312, 2.3456, 3.1415926)
@@ -89,6 +199,29 @@
     return s1, s2, s3, s4
 
 
+def format_c_numbers_unsigned(unsigned char c, unsigned short s, unsigned int n, unsigned long l):
+    """
+    >>> s1, s2, s3 = format_c_numbers_unsigned(123, 135, 12, 12312312)
+    >>> print(s1)
+    123 135 5675737012
+    >>> print(s2)
+      12f
+    >>> print(s3)
+    0C014    bbdef8
+
+    """
+    s1 = f"{c}{s:4} {l:o}{n}"
+    assert isinstance(s1, unicode), type(s1)
+    s2 = f"{n:-4}f"
+    assert isinstance(s2, unicode), type(s2)
+    s3 = f"{n:02X}{n:03o}{l:10x}"
+    assert isinstance(s3, unicode), type(s3)
+    return s1, s2, s3
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
 def format_c_numbers_max(int n, long l):
     """
     >>> n, l = max_int, max_long
@@ -112,25 +245,78 @@
     return s1, s2
 
 
+def format_c_number_const():
+    """
+    >>> s = format_c_number_const()
+    >>> s == '{0}'.format(max_long) or s
+    True
+    """
+    return f"{LONG_MAX}"
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
 def format_c_number_range(int n):
     """
-    >>> for i in range(-1000, 1000):
+    >>> for i in range(-1000, 1001):
     ...     assert format_c_number_range(i) == str(i)
     """
     return f'{n}'
 
 
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
 def format_c_number_range_width(int n):
     """
-    >>> for i in range(-1000, 1000):
-    ...     assert format_c_number_range_width(i) == '%04d' % i, format_c_number_range_width(i)
+    >>> for i in range(-1000, 1001):
+    ...     formatted = format_c_number_range_width(i)
+    ...     expected = '{n:04d}'.format(n=i)
+    ...     assert formatted == expected, "%r != %r" % (formatted, expected)
     """
     return f'{n:04}'
 
 
+def format_c_number_range_width0(int n):
+    """
+    >>> for i in range(-100, 101):
+    ...     formatted = format_c_number_range_width0(i)
+    ...     expected = '{n:00d}'.format(n=i)
+    ...     assert formatted == expected, "%r != %r" % (formatted, expected)
+    """
+    return f'{n:00}'
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
+def format_c_number_range_width1(int n):
+    """
+    >>> for i in range(-100, 101):
+    ...     formatted = format_c_number_range_width1(i)
+    ...     expected = '{n:01d}'.format(n=i)
+    ...     assert formatted == expected, "%r != %r" % (formatted, expected)
+    """
+    return f'{n:01}'
+
+
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
+def format_c_number_range_width_m4(int n):
+    """
+    >>> for i in range(-100, 101):
+    ...     formatted = format_c_number_range_width_m4(i)
+    ...     expected = '{n:-4d}'.format(n=i)
+    ...     assert formatted == expected, "%r != %r" % (formatted, expected)
+    """
+    return f'{n:-4}'
+
+
 def format_c_number_range_dyn_width(int n, int width):
     """
-    >>> for i in range(-1000, 1000):
+    >>> for i in range(-1000, 1001):
     ...     assert format_c_number_range_dyn_width(i, 0) == str(i), format_c_number_range_dyn_width(i, 0)
     ...     assert format_c_number_range_dyn_width(i, 1) == '%01d' % i, format_c_number_range_dyn_width(i, 1)
     ...     assert format_c_number_range_dyn_width(i, 4) == '%04d' % i, format_c_number_range_dyn_width(i, 4)
@@ -140,6 +326,9 @@
     return f'{n:0{width}}'
 
 
+@cython.test_fail_if_path_exists(
+    "//CoerceToPyTypeNode",
+)
 def format_bool(bint x):
     """
     >>> a, b, c, d = format_bool(1)
@@ -321,3 +510,108 @@
     b = f'x{value!s:6}x'
     assert isinstance(b, unicode), type(b)
     return a, b
+
+
+@cython.test_fail_if_path_exists(
+    "//FormattedValueNode",  # bytes.decode() returns unicode => formatting is useless
+    "//JoinedStrNode",       # replaced by call to PyUnicode_Concat()
+    "//PythonCapiCallNode//PythonCapiCallNode",
+)
+def format_decoded_bytes(bytes value):
+    """
+    >>> print(format_decoded_bytes(b'xyz'))
+    U-xyz
+    """
+    return f"U-{value.decode('utf-8')}"
+
+
+@cython.test_fail_if_path_exists(
+    "//AddNode",
+    "//ModNode",
+)
+@cython.test_assert_path_exists(
+    "//FormattedValueNode",
+    "//JoinedStrNode",
+)
+def generated_fstring(int i, float f, unicode u not None, o):
+    """
+    >>> i, f, u, o = 11, 1.3125, u'xyz', [1]
+    >>> print(((
+    ...     u"(i) %s-%.3s-%r-%.3r-%d-%3d-%-3d-%o-%04o-%x-%4x-%X-%03X-%.1f-%04.2f %% "
+    ...     u"(u) %s-%.2s-%r-%.7r-%05s-%-5s %% "
+    ...     u"(o) %s-%.2s-%r-%.2r %% "
+    ...     u"(f) %.2f-%d"
+    ... ) % (
+    ...     i, i, i, i, i, i, i, i, i, i, i, i, i, i, i,
+    ...     u, u, u, u, u, u,
+    ...     o, o, o, o,
+    ...     f, f,
+    ... )).replace("-u'xyz'", "-'xyz'"))
+    (i) 11-11-11-11-11- 11-11 -13-0013-b-   b-B-00B-11.0-11.00 % (u) xyz-xy-'xyz'-'xyz'-  xyz-xyz   % (o) [1]-[1-[1]-[1 % (f) 1.31-1
+
+    >>> print(generated_fstring(i, f, u, o).replace("-u'xyz'", "-'xyz'"))
+    (i) 11-11-11-11-11- 11-11 -13-0013-b-   b-B-00B-11.0-11.00 % (u) xyz-xy-'xyz'-'xyz'-  xyz-xyz   % (o) [1]-[1-[1]-[1 % (f) 1.31-1
+    """
+    return (
+        u"(i) %s-%.3s-%r-%.3r-%d-%3d-%-3d-%o-%04o-%x-%4x-%X-%03X-%.1f-%04.2f %% "
+        u"(u) %s-%.2s-%r-%.7r-%05s-%-5s %% "
+        u"(o) %s-%.2s-%r-%.2r %% "
+        u"(f) %.2f-%d"
+    ) % (
+        i, i, i, i, i, i, i, i, i, i, i, i, i, i, i,
+        u, u, u, u, u, u,
+        o, o, o, o,
+        f, f,
+    )
+
+
+@cython.test_assert_path_exists(
+    "//FormattedValueNode",
+    "//JoinedStrNode",
+)
+def percent_s_unicode(u, int i):
+    u"""
+    >>> u = u'x\u0194z'
+    >>> print(percent_s_unicode(u, 12))
+    x\u0194z-12
+    """
+    return u"%s-%d" % (u, i)
+
+
+@cython.test_assert_path_exists(
+    "//FormattedValueNode",
+)
+def sideeffect(l):
+    """
+    >>> class Listish(list):
+    ...     def __format__(self, format_spec):
+    ...         self.append("format called")
+    ...         return repr(self)
+    ...     def append(self, item):
+    ...         list.append(self, item)
+    ...         return self
+
+    >>> l = Listish()
+    >>> sideeffect(l)  if getattr(sys, 'pypy_version_info', ())[:2] != (7,3) else [123, 'format called']   # 7.3.4, 7.3.5
+    [123, 'format called']
+    """
+    f"{l.append(123)}"  # unused f-string !
+    return list(l)
+
+
+########################################
+# await inside f-string
+
+def test_await_inside_f_string():
+    """
+    >>> test_await_inside_f_string()
+    PARSED_SUCCESSFULLY
+    """
+
+    async def f():
+        return "some value"
+
+    async def main():
+        print(f"{await f()}")
+
+    print("PARSED_SUCCESSFULLY")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/funcexc_iter_T228.pyx cython-0.20.1+1~202203241016-9537/tests/run/funcexc_iter_T228.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/funcexc_iter_T228.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/funcexc_iter_T228.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 228
+# ticket: t228
 
 __doc__ = u"""
 >>> def py_iterator():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/function_as_method_py_T494.py cython-0.20.1+1~202203241016-9537/tests/run/function_as_method_py_T494.py
--- cython-0.20.1+1~201611251650-6686/tests/run/function_as_method_py_T494.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/function_as_method_py_T494.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 494
+# ticket: t494
 
 __doc__ = """
     >>> A.foo = foo
@@ -11,3 +11,35 @@
 
 def foo(self):
     return self is not None
+
+
+# assignment of functions used in a "static method" type way behaves differently
+# in Python2 and 3
+import sys
+if sys.version_info[0] == 2:
+    __doc__ = u"""
+>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL
+Traceback (most recent call last):
+    ...
+TypeError: unbound
+>>> C.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL
+Traceback (most recent call last):
+    ...
+TypeError: unbound
+"""
+else:
+    __doc__ = u"""
+>>> B.plus1(1)
+2
+>>> C.plus1(1)
+2
+"""
+
+def f_plus(a):
+    return a + 1
+
+class B:
+    plus1 = f_plus
+
+class C(object):
+    plus1 = f_plus
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/function_as_method_T494.pyx cython-0.20.1+1~202203241016-9537/tests/run/function_as_method_T494.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/function_as_method_T494.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/function_as_method_T494.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 494
+# ticket: t494
 # cython: binding=True
 
 __doc__ = """
@@ -12,3 +12,39 @@
 
 def foo(self):
     return self is not None
+
+# assignment of functions used in a "static method" type way behaves differently
+# in Python2 and 3
+import sys
+if sys.version_info[0] == 2:
+    __doc__ = """>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL
+Traceback (most recent call last):
+    ...
+TypeError: unbound
+"""
+else:
+    __doc__ = """>>> B.plus1(1)
+2
+"""
+
+# with binding==False assignment of functions always worked - doesn't match Python
+# behaviour but ensures Cython behaviour stays consistent
+__doc__ += """
+>>> B.plus1_nobind(1)
+2
+"""
+
+cimport cython
+
+def f_plus(a):
+    return a + 1
+
+@cython.binding(False)
+def f_plus_nobind(a):
+    return a+1
+
+cdef class B:
+    plus1 = f_plus
+    plus1_nobind = f_plus_nobind
+
+
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/function_binding_T494.pyx cython-0.20.1+1~202203241016-9537/tests/run/function_binding_T494.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/function_binding_T494.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/function_binding_T494.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 494
+# ticket: t494
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/function_self.py cython-0.20.1+1~202203241016-9537/tests/run/function_self.py
--- cython-0.20.1+1~201611251650-6686/tests/run/function_self.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/function_self.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,94 @@
+# mode: run
+# tag: pure2.7
+
+# cython: binding=True
+
+import cython
+import sys
+
+def regular(x):
+    """
+    >>> hasattr(regular, "__self__")
+    False
+    >>> nested = regular(10)
+    >>> hasattr(nested, "__self__")
+    False
+    """
+    def nested(y):
+        return x+y
+    return nested
+
+@cython.locals(x=cython.floating)
+def fused(x):
+    """
+    >>> nested = fused(10.)
+    >>> hasattr(nested, "__self__")
+    False
+
+    #>>> hasattr(fused, "__self__")  # FIXME this fails for fused functions
+    #False
+    # but this is OK:
+    >>> fused.__self__  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    AttributeError: 'function' object has no attribute '__self__'...
+    """
+    def nested_in_fused(y):
+        return x+y
+    return nested_in_fused
+
+# FIXME - doesn't currently work at all
+#def get_nested_fused(x):
+#    @cython.locals(x=cython.floating)
+#    def nested_fused(y):
+#        return x+y
+#    return nested_fused
+
+class C:
+    """
+    >>> c = C()
+    >>> c.regular.__self__ is c
+    True
+    >>> c.fused.__self__ is c
+    True
+    """
+    def regular(self):
+        pass
+
+    @cython.locals(x=cython.floating)
+    def fused(self, x):
+        return x
+
+__doc__ = ""
+if sys.version_info[0] > 2 or cython.compiled:
+    __doc__ += """
+    >>> hasattr(C.regular, "__self__")  # __self__==None on pure-python 2
+    False
+
+    # returns None on pure-python 2
+    >>> C.fused.__self__  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    AttributeError: 'function' object has no attribute '__self__'...
+    """
+
+if cython.compiled:
+    __doc__ = """
+    >>> fused['double'].__self__   #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    AttributeError: 'function' object has no attribute '__self__'...
+
+    >>> C.fused['double'].__self__   #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    AttributeError: 'function' object has no attribute '__self__'...
+
+    >>> c = C()
+    >>> c.fused['double'].__self__ is c   #doctest: +ELLIPSIS
+    True
+
+    # The PR that changed __self__ also changed how __doc__ is set up slightly
+    >>> fused['double'].__doc__ == fused.__doc__ and isinstance(fused.__doc__, str)
+    True
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_bound_functions.py cython-0.20.1+1~202203241016-9537/tests/run/fused_bound_functions.py
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_bound_functions.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_bound_functions.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,205 @@
+# mode: run
+# tag: pure3.0
+# cython: binding=True
+
+"""
+Test that fused functions can be used in the same way as CyFunctions with respect to
+assigning them to class attributes. Previously they enforced extra type/argument checks
+beyond those which CyFunctions did.
+"""
+
+import cython
+
+MyFusedClass = cython.fused_type(
+    float,
+    'Cdef',
+    object)
+
+def fused_func(x: MyFusedClass):
+    return (type(x).__name__, cython.typeof(x))
+
+IntOrFloat = cython.fused_type(int, float)
+
+def fused_func_0(x: IntOrFloat = 0):
+    """
+    Fused functions can legitimately take 0 arguments
+    >>> fused_func_0()
+    ('int', 'int')
+
+    # subscripted in module __doc__ conditionally
+    """
+    return (type(x).__name__, cython.typeof(x))
+
+def regular_func(x):
+    return (type(x).__name__, cython.typeof(x))
+
+def regular_func_0():
+    return
+
+@classmethod
+def fused_classmethod_free(cls, x: IntOrFloat):
+    return (cls.__name__, type(x).__name__)
+
+@cython.cclass
+class Cdef:
+    __doc__ = """
+    >>> c = Cdef()
+
+    # functions are callable with an instance of c
+    >>> c.fused_func()
+    ('Cdef', 'Cdef')
+    >>> c.regular_func()
+    ('Cdef', '{typeofCdef}')
+    >>> c.fused_in_class(1.5)
+    ('float', 'float')
+
+    # Fused functions are callable without an instance
+    # (This applies to everything in Py3 - see __doc__ below)
+    >>> Cdef.fused_func(1.5)
+    ('float', 'float')
+    >>> Cdef.fused_in_class(c, 1.5)
+    ('float', 'float')
+    >>> Cdef.fused_func_0()
+    ('int', 'int')
+
+    # Functions not expecting an argument don't work with an instance
+    >>> c.regular_func_0()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: regular_func_0() takes ... arguments ...1... given...
+
+    # Looking up a class attribute doesn't go through all of __get__
+    >>> Cdef.fused_in_class is Cdef.fused_in_class
+    True
+
+    # looking up a classmethod does go through __get__ though
+    >>> Cdef.fused_classmethod is Cdef.fused_classmethod
+    False
+    >>> Cdef.fused_classmethod_free is Cdef.fused_classmethod_free
+    False
+    >>> Cdef.fused_classmethod(1)
+    ('Cdef', 'int')
+    >>> Cdef.fused_classmethod_free(1)
+    ('Cdef', 'int')
+    """.format(typeofCdef = 'Python object' if cython.compiled else 'Cdef')
+
+    if cython.compiled:
+        __doc__ += """
+
+    # fused_func_0 does not accept a "Cdef" instance
+    >>> c.fused_func_0()
+    Traceback (most recent call last):
+    TypeError: No matching signature found
+
+    # subscripting requires fused methods (so  not pure Python)
+    >>> Cdef.fused_func_0['float']()
+    ('float', 'float')
+    >>> c.fused_func_0['float']()  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    TypeError: (Exception looks quite different in Python2 and 3 so no way to match both)
+
+    >>> Cdef.fused_classmethod['float'] is Cdef.fused_classmethod['float']
+    False
+    >>> Cdef.fused_classmethod_free['float'] is Cdef.fused_classmethod_free['float']
+    False
+    """
+    fused_func = fused_func
+    fused_func_0 = fused_func_0
+    regular_func = regular_func
+    regular_func_0 = regular_func_0
+
+    fused_classmethod_free = fused_classmethod_free
+
+    def fused_in_class(self, x: MyFusedClass):
+        return (type(x).__name__, cython.typeof(x))
+
+    def regular_in_class(self):
+        return type(self).__name__
+
+    @classmethod
+    def fused_classmethod(cls, x: IntOrFloat):
+        return (cls.__name__, type(x).__name__)
+
+class Regular(object):
+    __doc__ = """
+    >>> c = Regular()
+
+    # Functions are callable with an instance of C
+    >>> c.fused_func()
+    ('Regular', '{typeofRegular}')
+    >>> c.regular_func()
+    ('Regular', '{typeofRegular}')
+
+    # Fused functions are callable without an instance
+    # (This applies to everything in Py3 - see __doc__ below)
+    >>> Regular.fused_func(1.5)
+    ('float', 'float')
+    >>> Regular.fused_func_0()
+    ('int', 'int')
+
+    # Functions not expecting an argument don't work with an instance
+    >>> c.regular_func_0()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: regular_func_0() takes ... arguments ...1... given...
+
+    # Looking up a class attribute doesn't go through all of __get__
+    >>> Regular.fused_func is Regular.fused_func
+    True
+
+    # looking up a classmethod does go __get__ though
+    >>> Regular.fused_classmethod is Regular.fused_classmethod
+    False
+    >>> Regular.fused_classmethod_free is Regular.fused_classmethod_free
+    False
+    >>> Regular.fused_classmethod(1)
+    ('Regular', 'int')
+    >>> Regular.fused_classmethod_free(1)
+    ('Regular', 'int')
+    """.format(typeofRegular = "Python object" if cython.compiled else 'Regular')
+    if cython.compiled:
+        __doc__ += """
+    # fused_func_0 does not accept a "Regular" instance
+    >>> c.fused_func_0()
+    Traceback (most recent call last):
+    TypeError: No matching signature found
+
+    # subscripting requires fused methods (so  not pure Python)
+    >>> c.fused_func_0['float']()  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    TypeError: (Exception looks quite different in Python2 and 3 so no way to match both)
+    >>> Regular.fused_func_0['float']()
+    ('float', 'float')
+
+    >>> Regular.fused_classmethod['float'] is Regular.fused_classmethod['float']
+    False
+    >>> Regular.fused_classmethod_free['float'] is Regular.fused_classmethod_free['float']
+    False
+    """
+
+    fused_func = fused_func
+    fused_func_0 = fused_func_0
+    regular_func = regular_func
+    regular_func_0 = regular_func_0
+
+    fused_classmethod_free = fused_classmethod_free
+
+    @classmethod
+    def fused_classmethod(cls, x: IntOrFloat):
+        return (cls.__name__, type(x).__name__)
+
+import sys
+if sys.version_info[0] > 2:
+    # extra Py3 only tests - shows that functions added to a class can be called
+    # with an type as the first argument
+    __doc__ = """
+    >>> Cdef.regular_func(1.5)
+    ('float', '{typeoffloat}')
+    >>> Regular.regular_func(1.5)
+    ('float', '{typeoffloat}')
+    >>> Cdef.regular_func_0()
+    >>> Regular.regular_func_0()
+    """.format(typeoffloat='Python object' if cython.compiled else 'float')
+if cython.compiled:
+    __doc__ += """
+    >>> fused_func_0['float']()
+    ('float', 'float')
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_cmethods.srctree cython-0.20.1+1~202203241016-9537/tests/run/fused_cmethods.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_cmethods.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_cmethods.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,208 @@
+"""
+PYTHON setup.py build_ext -i
+PYTHON main.py
+"""
+
+######## main.py ########
+
+from __future__ import absolute_import
+from pkg.user import UseRegisters
+
+def test():
+    from pkg import called
+    assert called == [], called
+
+    ureg = UseRegisters()
+
+    assert called == [
+        'Before setFullFlags',
+        'setFullFlags was called',
+        'After setFullFlags',
+    ], called
+
+    del called[:]
+    ureg.call_write()
+
+    assert called == [
+        'Before regWriteWithOpWords',
+        'regWriteWithOpWords was called',
+        'regWriteWithOpWords leave function',
+        'After regWriteWithOpWords',
+    ], called
+
+    del called[:]
+    ureg.call_non_fused()
+
+    assert called == [
+        'Before nonFusedMiddle',
+        'nonFusedMiddle was called',
+        'After nonFusedMiddle',
+        'Before nonFusedBottom',
+        'nonFusedBottom was called',
+        'After nonFusedBottom',
+        'Before nonFusedTop',
+        'nonFusedTop was called',
+        'After nonFusedTop',
+    ], called
+
+
+def test_sub():
+    from pkg import called
+    from pkg.registers import SubRegisters
+    ureg = UseRegisters(reg_type=SubRegisters)
+
+    del called[:]
+    ureg.call_sub()
+
+    assert called == [
+        'Before nonFusedSub',
+        'nonFusedSub was called',
+        'After nonFusedSub',
+        'Before fusedSub',
+        'fusedSub was called',
+        'After fusedSub',
+    ], called
+
+
+test()
+test_sub()
+
+
+######## setup.py ########
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize('pkg/*.pyx'))
+
+######## pkg/__init__.py ########
+
+called = []
+
+######## pkg/user.pxd ########
+
+from .registers cimport Registers, SubRegisters
+
+cdef class UseRegisters:
+    cdef Registers registers
+
+
+######## pkg/user.pyx ########
+
+from . import called
+
+cdef class UseRegisters:
+    def __init__(self, reg_type=Registers):
+        self.registers = reg_type()
+        called.append("Before setFullFlags")
+        self.registers.setFullFlags(12345, 0)
+        called.append("After setFullFlags")
+
+    def call_write(self):
+        called.append("Before regWriteWithOpWords")
+        self.registers.regWriteWithOpWords(0, 0)
+        called.append("After regWriteWithOpWords")
+
+    def call_non_fused(self):
+        called.append("Before nonFusedMiddle")
+        self.registers.nonFusedMiddle(0, 0)
+        called.append("After nonFusedMiddle")
+
+        called.append("Before nonFusedBottom")
+        self.registers.nonFusedBottom(0, 0)
+        called.append("After nonFusedBottom")
+
+        called.append("Before nonFusedTop")
+        self.registers.nonFusedTop(0, 0)
+        called.append("After nonFusedTop")
+
+    def call_sub(self):
+        assert isinstance(self.registers, SubRegisters), type(self.registers)
+        called.append("Before nonFusedSub")
+        (self.registers).nonFusedSub(0, 0)
+        called.append("After nonFusedSub")
+
+        called.append("Before fusedSub")
+        (self.registers).fusedSub(0, 0)
+        called.append("After fusedSub")
+
+
+######## pkg/registers.pxd ########
+
+from cython cimport integral
+
+cdef class Registers:
+    cdef unsigned long long regs[1]
+    cdef void nonFusedTop(self, unsigned short regId, unsigned int value)
+    cdef void regWriteWithOpWords(self, unsigned short regId, integral value)
+    cdef void nonFusedMiddle(self, unsigned short regId, unsigned int value)
+    cdef void setFullFlags(self, integral reg0, unsigned int reg1)
+    cdef void nonFusedBottom(self, unsigned short regId, unsigned int value)
+    cdef void lastFusedImplFirst(self, integral reg0, unsigned int reg1)
+
+
+cdef class SubRegisters(Registers):
+    cdef void fusedSub(self, integral reg0, unsigned int reg1)
+    cdef void nonFusedSub(self, unsigned short regId, unsigned int value)
+
+
+######## pkg/registers.pyx ########
+
+from . import called
+
+cdef class Registers:
+    def __init__(self):
+        pass
+
+    cdef void lastFusedImplFirst(self, integral reg0, unsigned int reg1):
+        called.append("lastFusedImplFirst was called")
+
+    cdef void nonFusedTop(self, unsigned short regId, unsigned int value):
+        called.append("nonFusedTop was called")
+
+    cdef void regWriteWithOpWords(self, unsigned short regId, integral value):
+        called.append("regWriteWithOpWords was called")
+        self.regs[regId] = value
+        called.append("regWriteWithOpWords leave function")
+
+    cdef void nonFusedMiddle(self, unsigned short regId, unsigned int value):
+        called.append("nonFusedMiddle was called")
+
+    cdef void setFullFlags(self, integral reg0, unsigned int reg1):
+        called.append("setFullFlags was called")
+
+    cdef void nonFusedBottom(self, unsigned short regId, unsigned int value):
+        called.append("nonFusedBottom was called")
+
+
+cdef class SubRegisters(Registers):
+    cdef void fusedSub(self, integral reg0, unsigned int reg1):
+        called.append("fusedSub was called")
+
+    cdef void nonFusedSub(self, unsigned short regId, unsigned int value):
+        called.append("nonFusedSub was called")
+
+
+######## pkg/sub.pxd ########
+
+from .registers cimport *
+
+cdef class SubSubRegisters(SubRegisters):
+    cdef void fusedSubSubFirst(self, integral reg0, unsigned int reg1)
+    cdef void nonFusedSubSub(self, unsigned short regId, unsigned int value)
+    cdef void fusedSubSubLast(self, integral reg0, unsigned int reg1)
+
+
+######## pkg/sub.pyx ########
+
+from . import called
+
+cdef class SubSubRegisters(SubRegisters):
+    cdef void fusedSubSubFirst(self, integral reg0, unsigned int reg1):
+        called.append("fusedSubSubFirst was called")
+
+    cdef void nonFusedSubSub(self, unsigned short regId, unsigned int value):
+        called.append("nonFusedSubSub was called")
+
+    cdef void fusedSubSubLast(self, integral reg0, unsigned int reg1):
+        called.append("fusedSubSubLast was called")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_cpdef.pyx cython-0.20.1+1~202203241016-9537/tests/run/fused_cpdef.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_cpdef.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_cpdef.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,13 +1,17 @@
+# cython: language_level=3str
+# mode: run
+
 cimport cython
+import sys, io
 
 cy = __import__("cython")
 
 cpdef func1(self, cython.integral x):
-    print "%s," % (self,),
+    print(f"{self},", end=' ')
     if cython.integral is int:
-        print 'x is int', x, cython.typeof(x)
+        print('x is int', x, cython.typeof(x))
     else:
-        print 'x is long', x, cython.typeof(x)
+        print('x is long', x, cython.typeof(x))
 
 
 class A(object):
@@ -16,6 +20,18 @@
     def __str__(self):
         return "A"
 
+cdef class B:
+    cpdef int meth(self, cython.integral x):
+        print(f"{self},", end=' ')
+        if cython.integral is int:
+            print('x is int', x, cython.typeof(x))
+        else:
+            print('x is long', x, cython.typeof(x))
+        return 0
+
+    def __str__(self):
+        return "B"
+
 pyfunc = func1
 
 def test_fused_cpdef():
@@ -32,23 +48,71 @@
     A, x is long 2 long
     A, x is long 2 long
     A, x is long 2 long
+    
+    B, x is long 2 long
     """
     func1[int](None, 2)
     func1[long](None, 2)
     func1(None, 2)
 
-    print
+    print()
 
     pyfunc[cy.int](None, 2)
     pyfunc(None, 2)
 
-    print
+    print()
 
     A.meth[cy.int](A(), 2)
     A.meth(A(), 2)
     A().meth[cy.long](2)
     A().meth(2)
 
+    print()
+
+    B().meth(2)
+
+
+midimport_run = io.StringIO()
+if sys.version_info.major < 3:
+    # Monkey-patch midimport_run.write to accept non-unicode strings under Python 2.
+    midimport_run.write = lambda c: io.StringIO.write(midimport_run, unicode(c))
+
+realstdout = sys.stdout
+sys.stdout = midimport_run
+
+try:
+    # Run `test_fused_cpdef()` during import and save the result for
+    #        `test_midimport_run()`.
+    test_fused_cpdef()
+except Exception as e:
+    midimport_run.write(f"{e!r}\n")
+finally:
+    sys.stdout = realstdout
+
+def test_midimport_run():
+    # At one point, dynamically calling fused cpdef functions during import
+    #        would fail because the type signature-matching indices weren't
+    #        yet initialized.
+    #        (See Compiler.FusedNode.FusedCFuncDefNode._fused_signature_index,
+    #        GH-3366.)
+    """
+    >>> test_midimport_run()
+    None, x is int 2 int
+    None, x is long 2 long
+    None, x is long 2 long
+    
+    None, x is int 2 int
+    None, x is long 2 long
+    
+    A, x is int 2 int
+    A, x is long 2 long
+    A, x is long 2 long
+    A, x is long 2 long
+    
+    B, x is long 2 long
+    """
+    print(midimport_run.getvalue(), end='')
+
 
 def assert_raise(func, *args):
     try:
@@ -70,23 +134,31 @@
     assert_raise(A.meth)
     assert_raise(A().meth[cy.int])
     assert_raise(A.meth[cy.int])
+    assert_raise(B().meth, 1, 2, 3)
+
+def test_nomatch():
+    """
+    >>> func1(None, ())
+    Traceback (most recent call last):
+    TypeError: No matching signature found
+    """
 
 ctypedef long double long_double
 
 cpdef multiarg(cython.integral x, cython.floating y):
     if cython.integral is int:
-        print "x is an int,",
+        print("x is an int,", end=' ')
     else:
-        print "x is a long,",
+        print("x is a long,", end=' ')
 
     if cython.floating is long_double:
-        print "y is a long double:",
+        print("y is a long double:", end=' ')
     elif float is cython.floating:
-        print "y is a float:",
+        print("y is a float:", end=' ')
     else:
-        print "y is a double:",
+        print("y is a double:", end=' ')
 
-    print x, y
+    print(x, y)
 
 def test_multiarg():
     """
@@ -94,7 +166,46 @@
     x is an int, y is a float: 1 2.0
     x is an int, y is a float: 1 2.0
     x is a long, y is a double: 4 5.0
+    >>> multiarg()
+    Traceback (most recent call last):
+    TypeError: Expected at least 2 arguments, got 0
+    >>> multiarg(1, 2.0, 3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...2...arg...3...
     """
     multiarg[int, float](1, 2.0)
     multiarg[cy.int, cy.float](1, 2.0)
     multiarg(4, 5.0)
+
+def test_ambiguousmatch():
+    """
+    >>> multiarg(5, ())
+    Traceback (most recent call last):
+    TypeError: Function call with ambiguous argument types
+    >>> multiarg((), 2.0)
+    Traceback (most recent call last):
+    TypeError: Function call with ambiguous argument types
+    """
+
+# https://github.com/cython/cython/issues/4409
+# default arguments + fused cpdef were crashing
+cpdef literal_default(cython.integral x, some_string="value"):
+    return x, some_string
+
+cpdef mutable_default(cython.integral x, some_value=[]):
+    some_value.append(x)
+    return some_value
+
+def test_defaults():
+    """
+    >>> literal_default(1)
+    (1, 'value')
+    >>> literal_default(1, "hello")
+    (1, 'hello')
+    >>> mutable_default(1)
+    [1]
+    >>> mutable_default(2)
+    [1, 2]
+    >>> mutable_default(3,[])
+    [3]
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_cpp.pyx cython-0.20.1+1~202203241016-9537/tests/run/fused_cpp.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_cpp.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_cpp.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,6 +2,8 @@
 
 cimport cython
 from libcpp.vector cimport vector
+from libcpp.typeinfo cimport type_info
+from cython.operator cimport typeid
 
 def test_cpp_specialization(cython.floating element):
     """
@@ -14,3 +16,28 @@
     cdef vector[cython.floating] *v = new vector[cython.floating]()
     v.push_back(element)
     print cython.typeof(v), cython.typeof(element), v.at(0)
+
+cdef fused C:
+   int
+   object
+
+cdef const type_info* tidint = &typeid(int)
+def typeid_call(C x):
+    """
+    For GH issue 3203
+    >>> typeid_call(1)
+    True
+    """
+    cdef const type_info* a = &typeid(C)
+    return a[0] == tidint[0]
+
+cimport cython
+
+def typeid_call2(cython.integral x):
+    """
+    For GH issue 3203
+    >>> typeid_call2[int](1)
+    True
+    """
+    cdef const type_info* a = &typeid(cython.integral)
+    return a[0] == tidint[0]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_def.pyx cython-0.20.1+1~202203241016-9537/tests/run/fused_def.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_def.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_def.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -7,6 +7,9 @@
 cy = __import__("cython")
 cimport cython
 
+cdef extern from *:
+    int __Pyx_CyFunction_Check(object)
+
 cdef class Base(object):
     def __repr__(self):
         return type(self).__name__
@@ -51,19 +54,32 @@
 i = 9
 
 
-def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7):
+def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7,
+             another_opt = 2, yet_another_opt=3):
     """
-    Test runtime dispatch, indexing of various kinds and optional arguments
+    Test runtime dispatch, indexing of various kinds and optional arguments.
+    Use 5 arguments because at one point the optional argument from the
+    5th argument was overwriting that of the __pyx_fused dispatcher.
+    https://github.com/cython/cython/issues/3511
 
     >>> opt_func("spam", f, i)
     str object double long
     spam 5.60 9 5.60 9
+    >>> opt_func("spam", f, myi=i)
+    str object double long
+    spam 5.60 9 5.60 9
+    >>> opt_func("spam", myf=f, myi=i)
+    str object double long
+    spam 5.60 9 5.60 9
     >>> opt_func[str, float, int]("spam", f, i)
     str object float int
     spam 5.60 9 5.60 9
     >>> opt_func[str, cy.double, cy.long]("spam", f, i)
     str object double long
     spam 5.60 9 5.60 9
+    >>> opt_func[str, cy.double, cy.long]("spam", f, myi=i)
+    str object double long
+    spam 5.60 9 5.60 9
     >>> opt_func[str, float, cy.int]("spam", f, i)
     str object float int
     spam 5.60 9 5.60 9
@@ -105,16 +121,30 @@
 
     >>> opt_func(object(), f)
     Traceback (most recent call last):
-      ...
     TypeError: Function call with ambiguous argument types
+    >>> opt_func()
+    Traceback (most recent call last):
+    TypeError: Expected at least 1 argument, got 0
+    >>> opt_func("abc", f, i, 5, 5, 5)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...at most 5...
     >>> opt_func[ExtClassA, cy.float, cy.long](object(), f)
     Traceback (most recent call last):
-      ...
     TypeError: Argument 'obj' has incorrect type (expected fused_def.ExtClassA, got object)
     """
     print cython.typeof(obj), cython.typeof(myf), cython.typeof(myi)
     print obj, "%.2f" % myf, myi, "%.2f" % f, i
 
+def run_cyfunction_check():
+    """
+    tp_base of the fused function was being set incorrectly meaning
+    it wasn't being identified as a CyFunction
+    >>> run_cyfunction_check()
+    fused_cython_function
+    1
+    """
+    print(type(opt_func).__name__.rsplit('.', 1)[-1])
+    print(__Pyx_CyFunction_Check(opt_func))  # should be True
 
 def test_opt_func():
     """
@@ -125,6 +155,28 @@
     opt_func("ham", f, entry4)
 
 
+def test_opt_func_introspection():
+    """
+    >>> opt_func.__defaults__
+    (1.2, 7, 2, 3)
+    >>> opt_func.__kwdefaults__
+    >>> opt_func.__annotations__
+    {}
+
+    >>> opt_func[str, float, int].__defaults__
+    (1.2, 7, 2, 3)
+    >>> opt_func[str, float, int].__kwdefaults__
+    >>> opt_func[str, float, int].__annotations__
+    {}
+
+    >>> opt_func[str, cy.double, cy.long].__defaults__
+    (1.2, 7, 2, 3)
+    >>> opt_func[str, cy.double, cy.long].__kwdefaults__
+    >>> opt_func[str, cy.double, cy.long].__annotations__
+    {}
+    """
+
+
 def func_with_object(fused_with_object obj, cython.integral myi = 7):
     """
     >>> func_with_object(1)
@@ -370,3 +422,31 @@
     >>> test_decorators.order
     [3, 2, 1]
     """
+
+@cython.binding(True)
+def bind_me(self, cython.floating a=1.):
+    return a
+
+cdef class HasBound:
+    """
+    Using default arguments of bound specialized fused functions used to cause a segfault
+    https://github.com/cython/cython/issues/3370
+    >>> inst = HasBound()
+    >>> inst.func()
+    1.0
+    >>> inst.func(2)
+    2.0
+    >>> inst.func_fused()
+    1.0
+    >>> inst.func_fused(2.)
+    2.0
+    >>> bind_me.__defaults__
+    (1.0,)
+    >>> inst.func.__defaults__
+    (1.0,)
+    >>> inst.func_fused.__defaults__
+    (1.0,)
+    """
+    func = bind_me[float]
+
+    func_fused = bind_me
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/fused_types.pyx cython-0.20.1+1~202203241016-9537/tests/run/fused_types.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/fused_types.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/fused_types.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,5 @@
 # mode: run
+# ticket: t1772
 
 cimport cython
 from cython.view cimport array
@@ -20,6 +21,7 @@
 ctypedef int *p_int
 fused_type3 = cython.fused_type(int, double)
 fused_composite = cython.fused_type(fused_type2, fused_type3)
+just_float = cython.fused_type(float)
 
 def test_pure():
     """
@@ -119,7 +121,42 @@
     print
     print fused_with_pointer(string_array).decode('ascii')
 
-include "cythonarrayutil.pxi"
+cdef fused_type1* fused_pointer_except_null(fused_type1* x) except NULL:
+    if fused_type1 is string_t:
+        assert(bool(x[0]))
+    else:
+        assert(x[0] < 10)
+    return x
+
+def test_fused_pointer_except_null(value):
+    """
+    >>> test_fused_pointer_except_null(1)
+    1
+    >>> test_fused_pointer_except_null(2.0)
+    2.0
+    >>> test_fused_pointer_except_null(b'foo')
+    foo
+    >>> test_fused_pointer_except_null(16)
+    Traceback (most recent call last):
+    AssertionError
+    >>> test_fused_pointer_except_null(15.1)
+    Traceback (most recent call last):
+    AssertionError
+    >>> test_fused_pointer_except_null(b'')
+    Traceback (most recent call last):
+    AssertionError
+    """
+    if isinstance(value, int):
+        test_int = cython.declare(cython.int, value)
+        print fused_pointer_except_null(&test_int)[0]
+    elif isinstance(value, float):
+        test_float = cython.declare(cython.float, value)
+        print fused_pointer_except_null(&test_float)[0]
+    elif isinstance(value, bytes):
+        test_str = cython.declare(string_t, value)
+        print fused_pointer_except_null(&test_str)[0].decode('ascii')
+
+include "../testsupport/cythonarrayutil.pxi"
 
 cpdef cython.integral test_fused_memoryviews(cython.integral[:, ::1] a):
     """
@@ -291,6 +328,8 @@
     cdef cython.floating[:] otherarray = array[0:100:1]
     print cython.typeof(array), cython.typeof(otherarray), \
           array[5], otherarray[6]
+    cdef cython.floating value;
+    cdef cython.floating[:] test_cast = &value
 
 def test_fused_memslice_dtype_repeated(cython.floating[:] array1, cython.floating[:] array2):
     """
@@ -326,6 +365,22 @@
     """
     print cython.typeof(array1), cython.typeof(array2), cython.typeof(array3)
 
+def test_fused_const_memslice_dtype_repeated(const cython.floating[:] array1, cython.floating[:] array2):
+    """Test fused types memory view with one being const
+
+    >>> sorted(test_fused_const_memslice_dtype_repeated.__signatures__)
+    ['double', 'float']
+
+    >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(8, 'd'))
+    const double[:] double[:]
+    >>> test_fused_const_memslice_dtype_repeated(get_array(4, 'f'), get_array(4, 'f'))
+    const float[:] float[:]
+    >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(4, 'f'))
+    Traceback (most recent call last):
+    ValueError: Buffer dtype mismatch, expected 'double' but got 'float'
+    """
+    print cython.typeof(array1), cython.typeof(array2)
+
 def test_cython_numeric(cython.numeric arg):
     """
     Test to see whether complex numbers have their utility code declared
@@ -351,6 +406,18 @@
     """
     _test_index_fused_args[cython.floating, ints_t](f, i)
 
+cdef _test_index_const_fused_args(const cython.floating f, const ints_t i):
+    print(cython.typeof(f), cython.typeof(i))
+
+def test_index_const_fused_args(const cython.floating f, const ints_t i):
+    """Test indexing function implementation with const fused type args
+
+    >>> import cython
+    >>> test_index_const_fused_args[cython.double, cython.int](2.0, 3)
+    ('const double', 'const int')
+    """
+    _test_index_const_fused_args[cython.floating, ints_t](f, i)
+
 
 def test_composite(fused_composite x):
     """
@@ -365,3 +432,126 @@
         return x
     else:
         return 2 * x
+
+
+cdef cdef_func_const_fused_arg(const cython.floating val,
+                               const fused_type1 * ptr_to_const,
+                               const (cython.floating *) const_ptr):
+    print(val, cython.typeof(val))
+    print(ptr_to_const[0], cython.typeof(ptr_to_const[0]))
+    print(const_ptr[0], cython.typeof(const_ptr[0]))
+
+    ptr_to_const = NULL  # pointer is not const, value is const
+    const_ptr[0] = 0.0  # pointer is const, value is not const
+
+def test_cdef_func_with_const_fused_arg():
+    """Test cdef function with const fused type argument
+
+    >>> test_cdef_func_with_const_fused_arg()
+    (0.0, 'const float')
+    (1, 'const int')
+    (2.0, 'float')
+    """
+    cdef float arg0 = 0.0
+    cdef int arg1 = 1
+    cdef float arg2 = 2.0
+    cdef_func_const_fused_arg(arg0, &arg1, &arg2)
+
+
+cdef in_check_1(just_float x):
+    return just_float in floating
+
+cdef in_check_2(just_float x, floating y):
+    # the "floating" on the right-hand side of the in statement should not be specialized
+    # - the test should still work.
+    return just_float in floating
+
+cdef in_check_3(floating x):
+    # the floating on the left-hand side of the in statement should be specialized
+    # but the one of the right-hand side should not (so that the test can still work).
+    return floating in floating
+
+def test_fused_in_check():
+    """
+    It should be possible to use fused types on in "x in ...fused_type" statements
+    even if that type is specialized in the function.
+
+    >>> test_fused_in_check()
+    True
+    True
+    True
+    True
+    """
+    print(in_check_1(1.0))
+    print(in_check_2(1.0, 2.0))
+    print(in_check_2[float, double](1.0, 2.0))
+    print(in_check_3[float](1.0))
+
+
+### see GH3642 - presence of cdef inside "unrelated" caused a type to be incorrectly inferred
+cdef unrelated(cython.floating x):
+    cdef cython.floating t = 1
+    return t
+
+cdef handle_float(float* x): return 'float'
+
+cdef handle_double(double* x): return 'double'
+
+def convert_to_ptr(cython.floating x):
+    """
+    >>> convert_to_ptr(1.0)
+    'double'
+    >>> convert_to_ptr['double'](1.0)
+    'double'
+    >>> convert_to_ptr['float'](1.0)
+    'float'
+    """
+    if cython.floating is float:
+        return handle_float(&x)
+    elif cython.floating is double:
+        return handle_double(&x)
+
+cdef double get_double():
+    return 1.0
+cdef float get_float():
+    return 0.0
+
+cdef call_func_pointer(cython.floating (*f)()):
+    return f()
+
+def test_fused_func_pointer():
+    """
+    >>> test_fused_func_pointer()
+    1.0
+    0.0
+    """
+    print(call_func_pointer(get_double))
+    print(call_func_pointer(get_float))
+
+cdef double get_double_from_int(int i):
+    return i
+
+cdef call_func_pointer_with_1(cython.floating (*f)(cython.integral)):
+    return f(1)
+
+def test_fused_func_pointer2():
+    """
+    >>> test_fused_func_pointer2()
+    1.0
+    """
+    print(call_func_pointer_with_1(get_double_from_int))
+
+cdef call_function_that_calls_fused_pointer(object (*f)(cython.floating (*)(cython.integral))):
+    if cython.floating is double and cython.integral is int:
+        return 5*f(get_double_from_int)
+    else:
+        return None  # practically it's hard to make this kind of function useful...
+
+def test_fused_func_pointer_multilevel():
+    """
+    >>> test_fused_func_pointer_multilevel()
+    5.0
+    None
+    """
+    print(call_function_that_calls_fused_pointer(call_func_pointer_with_1[double, int]))
+    print(call_function_that_calls_fused_pointer(call_func_pointer_with_1[float, int]))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/future_division.pyx cython-0.20.1+1~202203241016-9537/tests/run/future_division.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/future_division.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/future_division.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -157,3 +157,73 @@
     (0.5, 2.0)
     """
     return a/b, b/a
+
+
+def div_by_0(a):
+    """
+    >>> div_by_0(0)
+    'OK'
+    >>> div_by_0(0.0)
+    'OK'
+    """
+    try:
+        1/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 1"
+    try:
+        1//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 2"
+    try:
+        5.0/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 3"
+    try:
+        5.0//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 4"
+    try:
+        5/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 5"
+    try:
+        5//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 6"
+    try:
+        (2**15)/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 7"
+    try:
+        (2**15)//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 8"
+    try:
+        (2**30)/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 9"
+    try:
+        (2**30)//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 10"
+    return 'OK'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/future_unicode_literals.pyx cython-0.20.1+1~202203241016-9537/tests/run/future_unicode_literals.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/future_unicode_literals.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/future_unicode_literals.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -9,6 +9,8 @@
     True
     >>> isinstance(b, bytes)
     True
+    >>> raw ==  'abc\\\\xf8\\\\t\\u00f8\\U000000f8'  # unescaped by Python (required by doctest)
+    True
 """
 else:
     __doc__ = u"""
@@ -18,9 +20,13 @@
     True
     >>> isinstance(b, str)
     True
+    >>> raw == u'abc\\\\xf8\\\\t\\u00f8\\U000000f8'  # unescaped by Python (required by doctest)
+    True
 """
 
 u = "test"
 
 cdef char* s = "bytes test"
 b = s
+
+raw = r'abc\xf8\t\u00f8\U000000f8'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generator_expressions_and_locals.pyx cython-0.20.1+1~202203241016-9537/tests/run/generator_expressions_and_locals.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/generator_expressions_and_locals.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generator_expressions_and_locals.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: genexpr, locals
-# ticket: 715
+# ticket: t715
 
 def genexpr_not_in_locals():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generator_frame_cycle.py cython-0.20.1+1~202203241016-9537/tests/run/generator_frame_cycle.py
--- cython-0.20.1+1~201611251650-6686/tests/run/generator_frame_cycle.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generator_frame_cycle.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,13 +1,9 @@
 # mode: run
 # tag: generator
 
+import cython
 import sys
 
-def _next(it):
-    if sys.version_info[0] >= 3:
-        return next(it)
-    else:
-        return it.next()
 
 def test_generator_frame_cycle():
     """
@@ -23,8 +19,42 @@
         finally:
             testit.append("I'm done")
     g = whoo()
-    _next(g)
+    next(g)
+
     # Frame object cycle
     eval('g.throw(ValueError)', {'g': g})
     del g
+
+    return tuple(testit)
+
+
+def test_generator_frame_cycle_with_outer_exc():
+    """
+    >>> test_generator_frame_cycle_with_outer_exc()
+    ("I'm done",)
+    """
+    testit = []
+    def whoo():
+        try:
+            yield
+        except:
+            yield
+        finally:
+            testit.append("I'm done")
+    g = whoo()
+    next(g)
+
+    try:
+        raise ValueError()
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+        # Frame object cycle
+        eval('g.throw(ValueError)', {'g': g})
+        # CPython 3.3 handles this incorrectly itself :)
+        if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+        del g
+        if cython.compiled or sys.version_info[:2] not in [(3, 2), (3, 3)]:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+
     return tuple(testit)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generators_GH1731.pyx cython-0.20.1+1~202203241016-9537/tests/run/generators_GH1731.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/generators_GH1731.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generators_GH1731.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,70 @@
+# mode: run
+# ticket: 1731
+
+
+def cygen():
+    yield 1
+
+
+def test_from_cython(g):
+    """
+    >>> def pygen(): yield 1
+    >>> test_from_cython(pygen)
+    Traceback (most recent call last):
+    ZeroDivisionError: integer division or modulo by zero
+
+    >>> test_from_cython(cygen)
+    Traceback (most recent call last):
+    ZeroDivisionError: integer division or modulo by zero
+    """
+    try:
+        1 / 0
+    except:
+        for _ in g():
+            pass
+        raise
+
+
+def test_from_python():
+    """
+    >>> def test(g):
+    ...     try:
+    ...         1 / 0
+    ...     except:
+    ...         for _ in g():
+    ...             pass
+    ...         raise
+
+    >>> def pygen():
+    ...     yield 1
+    >>> test(pygen)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+
+    >>> test(cygen)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+    """
+
+
+def test_from_console():
+    """
+    >>> def pygen(): yield 1
+    >>> try:  # doctest: +ELLIPSIS
+    ...     1 / 0
+    ... except:
+    ...     for _ in pygen():
+    ...         pass
+    ...     raise
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+
+    >>> try:  # doctest: +ELLIPSIS
+    ...     1 / 0
+    ... except:
+    ...     for _ in cygen():
+    ...         pass
+    ...     raise
+    Traceback (most recent call last):
+    ZeroDivisionError: ...division ...by zero
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generators_py35.py cython-0.20.1+1~202203241016-9537/tests/run/generators_py35.py
--- cython-0.20.1+1~201611251650-6686/tests/run/generators_py35.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generators_py35.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,40 @@
+# mode: run
+# tag: generators, pure3.5
+
+from __future__ import generator_stop
+
+# "generator_stop" was only added in Py3.5.
+
+
+def with_outer_raising(*args):
+    """
+    >>> x = with_outer_raising(1, 2, 3)
+    >>> try:
+    ...     list(x())
+    ... except RuntimeError:
+    ...     print("OK!")
+    ... else:
+    ...     print("NOT RAISED!")
+    OK!
+    """
+    def generator():
+        for i in args:
+            yield i
+        raise StopIteration
+    return generator
+
+
+def anno_gen(x: 'int') -> 'float':
+    """
+    >>> gen = anno_gen(2)
+    >>> next(gen)
+    2.0
+    >>> ret, arg = sorted(anno_gen.__annotations__.items())
+    >>> print(ret[0]); print(str(ret[1]).strip("'"))  # strip makes it pass with/without PEP563
+    return
+    float
+    >>> print(arg[0]); print(str(arg[1]).strip("'"))
+    x
+    int
+    """
+    yield float(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generators_py.py cython-0.20.1+1~202203241016-9537/tests/run/generators_py.py
--- cython-0.20.1+1~201611251650-6686/tests/run/generators_py.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generators_py.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,7 @@
 # mode: run
 # tag: generators
 
+import sys
 import cython
 
 
@@ -80,17 +81,6 @@
             yield i
     return generator
 
-def with_outer_raising(*args):
-    """
-    >>> x = with_outer_raising(1, 2, 3)
-    >>> list(x())
-    [1, 2, 3]
-    """
-    def generator():
-        for i in args:
-            yield i
-        raise StopIteration
-    return generator
 
 def test_close():
     """
@@ -147,25 +137,39 @@
         except ValueError:
             pass
 
+
 def check_yield_in_except():
     """
-    >>> import sys
-    >>> orig_exc = sys.exc_info()[0]
-    >>> g = check_yield_in_except()
-    >>> next(g)
-    >>> next(g)
-    >>> orig_exc is sys.exc_info()[0] or sys.exc_info()[0]
+    >>> if sys.version_info[0] == 2: sys.exc_clear()
+    >>> try:
+    ...     raise TypeError("RAISED !")
+    ... except TypeError as orig_exc:
+    ...     assert isinstance(orig_exc, TypeError), orig_exc
+    ...     g = check_yield_in_except()
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
+    ...     next(g)
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
+    ...     next(g)
+    ...     print(orig_exc is sys.exc_info()[1] or sys.exc_info())
+    True
+    True
     True
+    >>> next(g)
+    Traceback (most recent call last):
+    StopIteration
     """
     try:
         yield
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        if cython.compiled or sys.version_info[0] > 2:
+            assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def yield_in_except_throw_exc_type():
     """
-    >>> import sys
     >>> g = yield_in_except_throw_exc_type()
     >>> next(g)
     >>> g.throw(TypeError)
@@ -177,12 +181,14 @@
     """
     try:
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def yield_in_except_throw_instance():
     """
-    >>> import sys
     >>> g = yield_in_except_throw_instance()
     >>> next(g)
     >>> g.throw(TypeError())
@@ -194,8 +200,11 @@
     """
     try:
         raise ValueError
-    except ValueError:
+    except ValueError as exc:
+        assert sys.exc_info()[1] is exc, sys.exc_info()
         yield
+        assert sys.exc_info()[1] is exc, sys.exc_info()
+
 
 def test_swap_assignment():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/generators.pyx cython-0.20.1+1~202203241016-9537/tests/run/generators.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/generators.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/generators.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: generators
+# tag: generators, gh3265
 
 try:
     import backports_abc
@@ -502,3 +502,82 @@
     True
     """
     yield 1
+
+
+def test_generator_frame(a=1):
+    """
+    >>> gen = test_generator_frame()
+    >>> import types
+    >>> isinstance(gen.gi_frame, types.FrameType) or gen.gi_frame
+    True
+    >>> gen.gi_frame is gen.gi_frame  # assert that it's cached
+    True
+    >>> gen.gi_frame.f_code is not None
+    True
+    >>> code_obj = gen.gi_frame.f_code
+    >>> code_obj.co_argcount
+    1
+    >>> code_obj.co_varnames
+    ('a', 'b')
+    """
+    b = a + 1
+    yield b
+
+
+# GH Issue 3265 - **kwds could cause a crash in some cases due to not
+# handling NULL pointers (in testing it shows as a REFNANNY error).
+# This was on creation of the generator and
+# doesn't really require it to be iterated through:
+
+def some_function():
+    return 0
+
+
+def test_generator_kwds1(**kwargs):
+    """
+    >>> for a in test_generator_kwds1():
+    ...     print(a)
+    0
+    """
+    yield some_function(**kwargs)
+
+
+def test_generator_kwds2(**kwargs):
+    """
+    >>> for a in test_generator_kwds2():
+    ...     print(a)
+    0
+    """
+    yield 0
+
+
+def test_generator_kwds3(**kwargs):
+    """
+    This didn't actually crash before but is still worth a try
+    >>> len(list(test_generator_kwds3()))
+    0
+    >>> for a in test_generator_kwds3(a=1):
+    ...    print(a)
+    a
+    """
+    yield from kwargs.keys()
+
+
+def test_generator_frame(a=1):
+    """
+    >>> gen = test_generator_frame()
+    >>> import types
+    >>> isinstance(gen.gi_frame, types.FrameType) or gen.gi_frame
+    True
+    >>> gen.gi_frame is gen.gi_frame  # assert that it's cached
+    True
+    >>> gen.gi_frame.f_code is not None
+    True
+    >>> code_obj = gen.gi_frame.f_code
+    >>> code_obj.co_argcount
+    1
+    >>> code_obj.co_varnames
+    ('a', 'b')
+    """
+    b = a + 1
+    yield b
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/genexpr_iterable_lookup_T600.pyx cython-0.20.1+1~202203241016-9537/tests/run/genexpr_iterable_lookup_T600.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/genexpr_iterable_lookup_T600.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/genexpr_iterable_lookup_T600.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 600
+# ticket: t600
 # tag: genexpr
 # cython: language_level=3
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/genexpr_T491.pyx cython-0.20.1+1~202203241016-9537/tests/run/genexpr_T491.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/genexpr_T491.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/genexpr_T491.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 491
+# ticket: t491
 
 def test_genexpr():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/genexpr_T715.pyx cython-0.20.1+1~202203241016-9537/tests/run/genexpr_T715.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/genexpr_T715.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/genexpr_T715.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 715
+# ticket: t715
 # tag: genexpr, comprehension
 
 def t715(*items):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/__getattribute__.pyx cython-0.20.1+1~202203241016-9537/tests/run/__getattribute__.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/__getattribute__.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/__getattribute__.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,62 +1,107 @@
-__doc__ = u"""
-__getattribute__ and __getattr__ special methods for a single class.
-"""
+# mode: run
+
+# __getattribute__ and __getattr__ special methods for a single class.
+
 
 cdef class just_getattribute:
     """
     >>> a = just_getattribute()
+    >>> a.called
+    1
+    >>> a.called
+    2
     >>> a.bar
     'bar'
+    >>> a.called
+    4
     >>> a.invalid
     Traceback (most recent call last):
     AttributeError
+    >>> a.called
+    6
     """
+    cdef readonly int called
     def __getattribute__(self,n):
+        self.called += 1
         if n == 'bar':
             return n
+        elif n == 'called':
+            return self.called
         else:
             raise AttributeError
 
+
 cdef class just_getattr:
     """
     >>> a = just_getattr()
+    >>> a.called
+    0
+    >>> a.called
+    0
     >>> a.foo
     10
+    >>> a.called
+    0
     >>> a.bar
     'bar'
+    >>> a.called
+    1
     >>> a.invalid
     Traceback (most recent call last):
     AttributeError
+    >>> a.called
+    2
     """
+    cdef readonly int called
     cdef readonly int foo
     def __init__(self):
         self.foo = 10
     def __getattr__(self,n):
+        self.called += 1
         if n == 'bar':
             return n
         else:
             raise AttributeError
 
+
 cdef class both:
     """
     >>> a = both()
+    >>> (a.called_getattr, a.called_getattribute)
+    (0, 2)
     >>> a.foo
     10
+    >>> (a.called_getattr, a.called_getattribute)
+    (0, 5)
     >>> a.bar
     'bar'
+    >>> (a.called_getattr, a.called_getattribute)
+    (1, 8)
     >>> a.invalid
     Traceback (most recent call last):
     AttributeError
+    >>> (a.called_getattr, a.called_getattribute)
+    (2, 11)
     """
+    cdef readonly int called_getattribute
+    cdef readonly int called_getattr
     cdef readonly int foo
     def __init__(self):
         self.foo = 10
+
     def __getattribute__(self,n):
+        self.called_getattribute += 1
         if n == 'foo':
             return self.foo
+        elif n == 'called_getattribute':
+            return self.called_getattribute
+        elif n == 'called_getattr':
+            return self.called_getattr
         else:
             raise AttributeError
+
     def __getattr__(self,n):
+        self.called_getattr += 1
         if n == 'bar':
             return n
         else:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/__getattribute_subclasses__.pyx cython-0.20.1+1~202203241016-9537/tests/run/__getattribute_subclasses__.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/__getattribute_subclasses__.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/__getattribute_subclasses__.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,105 +1,204 @@
-__doc__ = u"""
-__getattribute__ and __getattr__ special methods and subclasses.
+# mode: run
+
+# __getattribute__ and __getattr__ special methods and subclasses.
+
+cdef class boring:
+    cdef readonly int boring_member
+    cdef readonly int getattr_called
+    cdef int getattribute_called
+    def __init__(self):
+        self.boring_member = 10
+
+cdef class getattr_boring(boring):
+    """
+    getattr does not override members.
 
-getattr does not override members.
     >>> a = getattr_boring()
     >>> a.boring_member
     10
+    >>> a.getattr_called
+    0
     >>> print(a.resolved_by)
     getattr_boring
-
-getattribute does.
-    >>> a = getattribute_boring()
-    >>> a.boring_member
+    >>> a.getattr_called
+    1
+    >>> a.no_such_member
     Traceback (most recent call last):
     AttributeError
-    >>> print(a.resolved_by)
-    getattribute_boring
+    >>> a.getattr_called
+    2
+    """
+    def __getattr__(self,n):
+        self.getattr_called += 1
+        if n == 'resolved_by':
+            return 'getattr_boring'
+        elif n == 'getattr_boring':
+            return True
+        else:
+            raise AttributeError
 
-Is inherited.
-    >>> a = boring_boring_getattribute()
-    >>> a.boring_getattribute_member
-    Traceback (most recent call last):
-    AttributeError
-    >>> a.boring_boring_getattribute_member
-    Traceback (most recent call last):
-    AttributeError
-    >>> print(a.resolved_by)
-    _getattribute
 
-__getattribute__ is always tried first, then __getattr__, regardless of where
-in the inheritance hiarchy they came from.
-    >>> a = getattribute_boring_boring_getattr()
-    >>> a.foo
+# currently fails, see #1793
+#class getattr_boring_py(getattr_boring):
+#    __doc__ = getattr_boring.__doc__.replace(
+#        'getattr_boring()', 'getattr_boring_py()')
+
+
+cdef class getattribute_boring(boring):
+    """
+    getattribute overrides members.
+
+    >>> a = getattribute_boring()
+    >>> a.getattribute_called
+    1
+    >>> a.boring_member
     Traceback (most recent call last):
     AttributeError
+    >>> a.getattribute_called
+    3
     >>> print(a.resolved_by)
-    getattribute_boring_boring_getattr
-    >>> a.getattribute_boring_boring_getattr
-    True
-    >>> a._getattr
-    True
-
-    >>> a = getattr_boring_boring_getattribute()
-    >>> a.foo
+    getattribute_boring
+    >>> a.getattribute_called
+    5
+    >>> a.no_such_member
     Traceback (most recent call last):
     AttributeError
-    >>> print(a.resolved_by)
-    _getattribute
-    >>> a.getattr_boring_boring_getattribute
-    True
-    >>> a._getattribute
-    True
-
-"""
-
-cdef class boring:
-    cdef readonly int boring_member
-    def __init__(self):
-        self.boring_member = 10
-
-cdef class getattr_boring(boring):
-    def __getattr__(self,n):
-        if n == u'resolved_by':
-            return u'getattr_boring'
-        elif n == u'getattr_boring':
-            return True
-        else:
-            raise AttributeError
-
-cdef class getattribute_boring(boring):
+    >>> a.getattribute_called
+    7
+    """
     def __getattribute__(self,n):
-        if n == u'resolved_by':
-            return u'getattribute_boring'
-        elif n == u'getattribute_boring':
+        self.getattribute_called += 1
+        if n == 'resolved_by':
+            return 'getattribute_boring'
+        elif n == 'getattribute_boring':
             return True
+        elif n == 'getattribute_called':
+            return self.getattribute_called
         else:
             raise AttributeError
 
+
+class getattribute_boring_py(getattribute_boring):
+    __doc__ = getattribute_boring.__doc__.replace(
+        'getattribute_boring()', 'getattribute_boring_py()')
+
+
 cdef class _getattr:
+    cdef readonly int getattr_called
     def __getattr__(self,n):
-        if n == u'resolved_by':
-            return u'_getattr'
-        elif n == u'_getattr':
+        self.getattr_called += 1
+        if n == 'resolved_by':
+            return '_getattr'
+        elif n == '_getattr':
             return True
+        elif n == 'getattr_called':
+            # must only get here if __getattribute__ is overwritten
+            assert 'getattribute' in type(self).__name__
+            return self.getattr_called
         else:
             raise AttributeError
 
-cdef class _getattribute(boring):
+
+class getattr_py(_getattr):
+    """
+    getattr is inherited.
+
+    >>> a = getattr_py()
+    >>> a.getattr_called
+    0
+    >>> print(a.resolved_by)
+    _getattr
+    >>> a.getattr_called
+    1
+    >>> print(a._getattr)
+    True
+    >>> a.getattr_called
+    2
+    >>> a.no_such_member
+    Traceback (most recent call last):
+    AttributeError
+
+    # currently fails, see #1793
+    #>>> a.getattr_called
+    #3
+    """
+
+
+cdef class _getattribute:
+    cdef int getattribute_called
     def __getattribute__(self,n):
-        if n == u'resolved_by':
-            return u'_getattribute'
-        elif n == u'_getattribute':
+        self.getattribute_called += 1
+        if n == 'resolved_by':
+            return '_getattribute'
+        elif n == '_getattribute':
             return True
+        elif n == 'getattribute_called':
+            return self.getattribute_called
         else:
             raise AttributeError
 
+
+class getattribute_py(_getattribute):
+    """
+    getattribute is inherited.
+
+    >>> a = getattribute_py()
+    >>> a.getattribute_called
+    1
+    >>> print(a.resolved_by)
+    _getattribute
+    >>> a.getattribute_called
+    3
+    >>> print(a._getattribute)
+    True
+    >>> a.getattribute_called
+    5
+    >>> a.no_such_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> a.getattribute_called
+    7
+    """
+
+
 cdef class boring_getattribute(_getattribute):
     cdef readonly int boring_getattribute_member
 
 cdef class boring_boring_getattribute(boring_getattribute):
+    """
+    getattribute is inherited.
+
+    >>> a = boring_boring_getattribute()
+    >>> a.getattribute_called
+    1
+    >>> a.boring_getattribute_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> a.getattribute_called
+    3
+    >>> a.boring_boring_getattribute_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> a.getattribute_called
+    5
+    >>> print(a.resolved_by)
+    _getattribute
+    >>> a.getattribute_called
+    7
+    >>> a.no_such_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> a.getattribute_called
+    9
+    """
     cdef readonly int boring_boring_getattribute_member
 
+
+class boring_boring_getattribute_py(boring_boring_getattribute):
+    __doc__ = boring_boring_getattribute.__doc__.replace(
+        'boring_boring_getattribute()', 'boring_boring_getattribute_py()')
+
+
 cdef class boring_getattr(_getattr):
     cdef readonly int boring_getattr_member
 
@@ -107,19 +206,90 @@
     cdef readonly int boring_boring_getattr_member
 
 cdef class getattribute_boring_boring_getattr(boring_boring_getattr):
+    """
+    __getattribute__ is always tried first, then __getattr__, regardless of where
+    in the inheritance hierarchy they came from.
+
+    >>> a = getattribute_boring_boring_getattr()
+    >>> (a.getattr_called, a.getattribute_called)
+    (1, 2)
+    >>> print(a.resolved_by)
+    getattribute_boring_boring_getattr
+    >>> (a.getattr_called, a.getattribute_called)
+    (2, 5)
+    >>> a.getattribute_boring_boring_getattr
+    True
+    >>> (a.getattr_called, a.getattribute_called)
+    (3, 8)
+    >>> a._getattr
+    True
+    >>> (a.getattr_called, a.getattribute_called)
+    (5, 11)
+    >>> a.no_such_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> (a.getattr_called, a.getattribute_called)
+    (7, 14)
+    """
+    cdef int getattribute_called
     def __getattribute__(self,n):
-        if n == u'resolved_by':
-            return u'getattribute_boring_boring_getattr'
-        elif n == u'getattribute_boring_boring_getattr':
+        self.getattribute_called += 1
+        if n == 'resolved_by':
+            return 'getattribute_boring_boring_getattr'
+        elif n == 'getattribute_boring_boring_getattr':
             return True
+        elif n == 'getattribute_called':
+            return self.getattribute_called
         else:
             raise AttributeError
 
+
+# currently fails, see #1793
+#class getattribute_boring_boring_getattr_py(getattribute_boring_boring_getattr):
+#    __doc__ = getattribute_boring_boring_getattr.__doc__.replace(
+#        'getattribute_boring_boring_getattr()', 'getattribute_boring_boring_getattr_py()')
+
+
 cdef class getattr_boring_boring_getattribute(boring_boring_getattribute):
+    """
+    __getattribute__ is always tried first, then __getattr__, regardless of where
+    in the inheritance hierarchy they came from.
+
+    >>> a = getattr_boring_boring_getattribute()
+    >>> (a.getattr_called, a.getattribute_called)
+    (1, 2)
+    >>> print(a.resolved_by)
+    _getattribute
+    >>> (a.getattr_called, a.getattribute_called)
+    (2, 5)
+    >>> a.getattr_boring_boring_getattribute
+    True
+    >>> (a.getattr_called, a.getattribute_called)
+    (4, 8)
+    >>> a._getattribute
+    True
+    >>> (a.getattr_called, a.getattribute_called)
+    (5, 11)
+    >>> a.no_such_member
+    Traceback (most recent call last):
+    AttributeError
+    >>> (a.getattr_called, a.getattribute_called)
+    (7, 14)
+    """
+    cdef readonly int getattr_called  # note: property will not be used due to __getattribute__()
     def __getattr__(self,n):
-        if n == u'resolved_by':
-            return u'getattr_boring_boring_getattribute'
-        elif n == u'getattr_boring_boring_getattribute':
+        self.getattr_called += 1
+        if n == 'resolved_by':
+            return 'getattr_boring_boring_getattribute'
+        elif n == 'getattr_boring_boring_getattribute':
             return True
+        elif n == 'getattr_called':
+            return self.getattr_called
         else:
             raise AttributeError
+
+
+# currently fails, see #1793
+#class getattr_boring_boring_getattribute_py(getattr_boring_boring_getattribute):
+#    __doc__ = getattr_boring_boring_getattribute.__doc__.replace(
+#        'getattr_boring_boring_getattribute()', 'getattr_boring_boring_getattribute_py()')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/hasattr.pyx cython-0.20.1+1~202203241016-9537/tests/run/hasattr.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/hasattr.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/hasattr.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,5 @@
+# mode: run
+
 class Foo:
     @property
     def foo(self):
@@ -9,6 +11,10 @@
     def baz(self):
         return int(1)/int(0)
 
+
+unicode_foo = u"foo"
+
+
 def wrap_hasattr(obj, name):
     """
     >>> wrap_hasattr(None, "abc")
@@ -17,6 +23,8 @@
     True
     >>> wrap_hasattr(Foo(), "foo")
     True
+    >>> wrap_hasattr(Foo(), unicode_foo)
+    True
     >>> wrap_hasattr(Foo(), "spam")
     False
     >>> wrap_hasattr(Foo(), "bar")
@@ -27,5 +35,9 @@
     ZeroDivisionError: ...
     >>> wrap_hasattr(Foo(), "baz")
     False
+    >>> hasattr(Foo(), None)   #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+       ...
+    TypeError: ...attribute name must be string...
     """
     return hasattr(obj, name)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/hash_T326.pyx cython-0.20.1+1~202203241016-9537/tests/run/hash_T326.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/hash_T326.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/hash_T326.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 326
+# ticket: t326
 # tag: hash
 
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/if_and_or.pyx cython-0.20.1+1~202203241016-9537/tests/run/if_and_or.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/if_and_or.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/if_and_or.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,119 @@
+# mode: run
+# tag: if, and, or
+
+def if_x(x):
+    """
+    >>> if_x(0)
+    2
+    >>> if_x(1)
+    1
+    """
+    if x:
+        return 1
+    else:
+        return 2
+
+def if_not(x):
+    """
+    >>> if_not(0)
+    1
+    >>> if_not(1)
+    2
+    """
+    if not x:
+        return 1
+    else:
+        return 2
+
+
+def if_and(a, b):
+    """
+    >>> if_and(3, 0)
+    2
+    >>> if_and(0, 3)
+    2
+    >>> if_and(0, 0)
+    2
+    >>> if_and(3, 3)
+    1
+    """
+    if a and b:
+        return 1
+    else:
+        return 2
+
+
+def if_not_and(a, b):
+    """
+    >>> if_not_and(3, 0)
+    1
+    >>> if_not_and(0, 3)
+    1
+    >>> if_not_and(0, 0)
+    1
+    >>> if_not_and(3, 3)
+    2
+    """
+    if not (a and b):
+        return 1
+    else:
+        return 2
+
+
+def if_or(a, b):
+    """
+    >>> if_or(3, 0)
+    1
+    >>> if_or(0, 3)
+    1
+    >>> if_or(0, 0)
+    2
+    >>> if_or(3, 3)
+    1
+    """
+    if a or b:
+        return 1
+    else:
+        return 2
+
+
+def if_not_or(a, b):
+    """
+    >>> if_not_or(3, 0)
+    2
+    >>> if_not_or(0, 3)
+    2
+    >>> if_not_or(0, 0)
+    1
+    >>> if_not_or(3, 3)
+    2
+    """
+    if not (a or b):
+        return 1
+    else:
+        return 2
+
+
+def if_and_or(a, b, c, d):
+    """
+    >>> if_and_or(3, 0, 0, 3)
+    1
+    >>> if_and_or(0, 3, 0, 3)
+    1
+    >>> if_and_or(0, 3, 3, 0)
+    1
+    >>> if_and_or(0, 3, 3, 0)
+    1
+    >>> if_and_or(0, 0, 0, 0)
+    2
+    >>> if_and_or(0, 3, 0, 0)
+    2
+    >>> if_and_or(0, 0, 3, 0)
+    2
+    >>> if_and_or(0, 0, 0, 3)
+    2
+    """
+    if (a or b) and (c or d):
+        return 1
+    else:
+        return 2
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/if_else_expr.pyx cython-0.20.1+1~202203241016-9537/tests/run/if_else_expr.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/if_else_expr.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/if_else_expr.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -55,3 +55,15 @@
     y = 0 if 1.0else 1
     z = 0 if 1.else 1
     return x, y, z
+
+
+from libc cimport math
+
+def test_cfunc_ptrs(double x, bint round_down):
+    """
+    >>> test_cfunc_ptrs(2.5, round_down=True)
+    2.0
+    >>> test_cfunc_ptrs(2.5, round_down=False)
+    3.0
+    """
+    return (math.floor if round_down else math.ceil)(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ifelseexpr_T267.pyx cython-0.20.1+1~202203241016-9537/tests/run/ifelseexpr_T267.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ifelseexpr_T267.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ifelseexpr_T267.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: condexpr
-# ticket: 267
+# ticket: t267
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/importas.pyx cython-0.20.1+1~202203241016-9537/tests/run/importas.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/importas.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/importas.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,14 @@
+# mode: run
+# tag: all_language_levels
+
 __doc__ = u"""
+>>> try: sys
+... except NameError: pass
+... else: print("sys was defined!")
+>>> try: distutils
+... except NameError: pass
+... else: print("distutils was defined!")
+
 >>> import sys as sous
 >>> import distutils.core as corey
 >>> from copy import deepcopy as copey
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/import_error_T734.py cython-0.20.1+1~202203241016-9537/tests/run/import_error_T734.py
--- cython-0.20.1+1~201611251650-6686/tests/run/import_error_T734.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/import_error_T734.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 734
+# ticket: t734
 
 def test_import_error():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/importfrom.pyx cython-0.20.1+1~202203241016-9537/tests/run/importfrom.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/importfrom.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/importfrom.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -68,6 +68,10 @@
     try:
         from sys import version_info as maxunicode
     except TypeError, e:
+        if getattr(sys, "pypy_version_info", None):
+            # translate message
+            if e.args[0].startswith("int() argument must be"):
+                e = "an integer is required"
         print(e)
 
     try:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/import_star.pyx cython-0.20.1+1~202203241016-9537/tests/run/import_star.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/import_star.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/import_star.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -10,7 +10,9 @@
 # (there used to be a problem getting Cython conversion code generated here)
 cdef MyStruct _no_such_name_ = MyStruct(1, 2, 3)
 
+from libc.math cimport M_PI
 
+# Danger ahead!
 from sys import *
 
 
@@ -39,3 +41,12 @@
     >>> assert pth is not None
     """
     return modules, path
+
+
+def test_cimported_pi():
+    """
+    >>> pi = test_cimported_pi()
+    >>> 3.14 < pi < 3.15
+    True
+    """
+    return M_PI
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/include_multiple_modules.srctree cython-0.20.1+1~202203241016-9537/tests/run/include_multiple_modules.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/include_multiple_modules.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/include_multiple_modules.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,31 @@
+PYTHON setup.py build_ext --inplace
+
+############# setup.py #############
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(
+    ext_modules = cythonize(["a.pyx", "b.pyx", "include_both.pyx"]),
+    )
+
+############# a.pyx ###############
+
+cdef public f():
+    pass
+    
+############# b.pyx ###############
+
+cdef public g():
+    pass
+    
+############# include_both.pyx ####
+
+# This is just checking that a and b don't duplicate any names
+# and thus it's possible to include them both in one place
+
+cdef extern from "a.h":
+    pass
+    
+cdef extern from "b.h":
+    pass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/index.pyx cython-0.20.1+1~202203241016-9537/tests/run/index.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/index.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/index.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -29,6 +29,9 @@
     >>> index_tuple((1,1,2,3,5), 100)
     Traceback (most recent call last):
     IndexError: tuple index out of range
+    >>> index_tuple((1,1,2,3,5), -7)
+    Traceback (most recent call last):
+    IndexError: tuple index out of range
     >>> index_tuple(None, 0)
     Traceback (most recent call last):
     TypeError: 'NoneType' object is not subscriptable
@@ -46,6 +49,9 @@
     >>> index_list([2,3,5,7,11,13,17,19], 100)
     Traceback (most recent call last):
     IndexError: list index out of range
+    >>> index_list([2,3,5,7,11,13,17,19], -10)
+    Traceback (most recent call last):
+    IndexError: list index out of range
     >>> index_list(None, 0)
     Traceback (most recent call last):
     TypeError: 'NoneType' object is not subscriptable
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/initial_file_path.srctree cython-0.20.1+1~202203241016-9537/tests/run/initial_file_path.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/initial_file_path.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/initial_file_path.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -29,8 +29,8 @@
     traceback.print_exc()
 
 def test():
-    print "FILE: ", initial_file
-    print "PATH: ", initial_path
+    print("FILE: ", initial_file)
+    print("PATH: ", initial_path)
     assert initial_path[0].endswith('my_test_package'), initial_path
     assert initial_file.endswith('__init__.py'), initial_file
     assert import_error is None, import_error
@@ -51,8 +51,8 @@
     traceback.print_exc()
 
 def test():
-    print "FILE: ", initial_file
-    print "PATH: ", initial_path
+    print("FILE: ", initial_file)
+    print("PATH: ", initial_path)
     assert initial_path[0].endswith('another'), initial_path
     assert initial_file.endswith('__init__.py'), initial_file
     assert import_error is None, import_error
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/inlinepxd.pyx cython-0.20.1+1~202203241016-9537/tests/run/inlinepxd.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/inlinepxd.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/inlinepxd.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,8 @@
+# mode: run
+# tag: inline, pxd
+
+# cython: wraparound = False
+
 __doc__ = u"""
 >>> f()
 3
@@ -5,6 +10,10 @@
 6
 >>> h()
 6
+>>> i()
+6
+>>> j()
+6
 """
 
 cimport inlinepxd_support
@@ -18,3 +27,18 @@
 
 def h():
     return my_add3(1, 2, 3)
+
+def i():
+    return my_add3(5)
+
+def j():
+    return my_add3(2, 4)
+
+def test_wraparound():
+    """
+    >>> test_wraparound()
+    1.0
+    """
+    # the wraparound directive from this scope should not affect the inline pxd
+    a = [ 0.0, 1.0 ]
+    return inlinepxd_support.index(a)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/inlinepxd_support.pxd cython-0.20.1+1~202203241016-9537/tests/run/inlinepxd_support.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/inlinepxd_support.pxd	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/inlinepxd_support.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,9 @@
 
-cdef inline int my_add(int a, int b, int c):
+cdef inline int my_add(int a, int b=1, int c=0):
     return a + b + c
+
+cdef inline index(list L):
+    # This function should *not* be affected by directives set in the outer scope, such as "wraparound".
+    # See https://github.com/cython/cython/issues/1071
+    return L[-1]
+
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/in_list_with_side_effects_T544.pyx cython-0.20.1+1~202203241016-9537/tests/run/in_list_with_side_effects_T544.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/in_list_with_side_effects_T544.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/in_list_with_side_effects_T544.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 544
+# ticket: t544
 
 def count(i=[0]):
     i[0] += 1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/int128.pyx cython-0.20.1+1~202203241016-9537/tests/run/int128.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/int128.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/int128.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -117,3 +117,73 @@
     """
     cdef int128_t n = x
     return n
+
+
+def get_int_distribution(shuffle=True):
+    """
+    >>> L = get_int_distribution()
+    >>> bigint(L[0])
+    682
+    >>> bigint(L[ len(L) // 2 ])
+    5617771410183435
+    >>> bigint(L[-1])
+    52818775009509558395695966805
+    >>> len(L)
+    66510
+    """
+    # Large integers that cover 1-4 (30 bits) or 1-7 (15 bits) PyLong digits.
+    # Uses only integer calculations to avoid rounding issues.
+    pow2 = [2**exp for exp in range(98)]
+    ints = [
+        n // 3
+        for i in range(11, len(pow2) - 1)
+        # Take a low but growing number of integers from each power-of-2 range.
+        for n in range(pow2[i], pow2[i+1], pow2[i - 8] - 1)
+    ]
+    return ints * 3  # longer list, but keeps median in the middle
+
+
+def intsum(L):
+    """
+    >>> L = get_int_distribution()
+    >>> bigint(intsum(L))
+    61084913298497804284622382871263
+    >>> bigint(sum(L))
+    61084913298497804284622382871263
+
+    >>> from random import shuffle
+    >>> shuffle(L)
+    >>> bigint(intsum(L))
+    61084913298497804284622382871263
+    """
+    cdef uint128_t i, x = 0
+    for i in L:
+        x += i
+    return x
+
+
+def intxor(L):
+    """
+    >>> L = get_int_distribution()
+    >>> bigint(intxor(L))
+    31773794341658093722410838161
+    >>> bigint(intxor(L * 2))
+    0
+    >>> import operator
+    >>> from functools import reduce
+    >>> bigint(reduce(operator.xor, L))
+    31773794341658093722410838161
+    >>> bigint(reduce(operator.xor, L * 2))
+    0
+
+    >>> from random import shuffle
+    >>> shuffle(L)
+    >>> bigint(intxor(L))
+    31773794341658093722410838161
+    >>> bigint(intxor(L * 2))
+    0
+    """
+    cdef uint128_t i, x = 0
+    for i in L:
+        x ^= i
+    return x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/intern_T431.pyx cython-0.20.1+1~202203241016-9537/tests/run/intern_T431.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/intern_T431.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/intern_T431.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 431
+# ticket: t431
 
 __doc__ = u"""
 >>> s == s_interned
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/int_float_builtins_as_casts_T400_long_double.pyx cython-0.20.1+1~202203241016-9537/tests/run/int_float_builtins_as_casts_T400_long_double.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/int_float_builtins_as_casts_T400_long_double.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/int_float_builtins_as_casts_T400_long_double.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,21 @@
+# ticket: t400
+
+cimport cython
+
+
+@cython.test_fail_if_path_exists("//SingleAssignmentNode//TypecastNode")
+@cython.test_assert_path_exists(
+    "//PythonCapiCallNode",
+    "//PythonCapiCallNode/PythonCapiFunctionNode/@cname = '__Pyx_truncl'",
+)
+def long_double_to_float_int(long double x):
+    """
+    >>> long_double_to_float_int(4.1)
+    4.0
+    >>> long_double_to_float_int(-4.1)
+    -4.0
+    >>> long_double_to_float_int(4)
+    4.0
+    """
+    cdef float r = int(x)
+    return r
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/int_float_builtins_as_casts_T400.pyx cython-0.20.1+1~202203241016-9537/tests/run/int_float_builtins_as_casts_T400.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/int_float_builtins_as_casts_T400.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/int_float_builtins_as_casts_T400.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 400
+# ticket: t400
 
 cimport cython
 
@@ -10,6 +10,9 @@
     4
     >>> double_to_short_int(4)
     4
+    >>> double_to_short_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef short r = int(x)
     return r
@@ -22,6 +25,9 @@
     4
     >>> double_to_pyssizet_int(4)
     4
+    >>> double_to_pyssizet_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef Py_ssize_t r = int(x)
     return r
@@ -34,6 +40,9 @@
     4
     >>> int_to_pyssizet_int(4)
     4
+    >>> int_to_pyssizet_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef Py_ssize_t r = int(x)
     return r
@@ -56,6 +65,9 @@
     """
     >>> int_to_short_int(4)
     4
+    >>> int_to_short_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...integer...
     """
     cdef short r = int(x)
     return r
@@ -66,6 +78,9 @@
     """
     >>> short_to_float_float(4)
     4.0
+    >>> short_to_float_float('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...integer...
     """
     cdef float r = float(x)
     return r
@@ -76,6 +91,9 @@
     """
     >>> short_to_double_float(4)
     4.0
+    >>> short_to_double_float('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...integer...
     """
     cdef double r = float(x)
     return r
@@ -86,6 +104,9 @@
     """
     >>> short_to_double_int(4)
     4.0
+    >>> short_to_double_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...integer...
     """
     cdef double r = int(x)
     return r
@@ -97,6 +118,9 @@
     True
     >>> float_to_float_float(4)
     4.0
+    >>> float_to_float_float('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef float r = float(x)
     return r
@@ -109,6 +133,9 @@
     True
     >>> double_to_double_float(4)
     4.0
+    >>> double_to_double_float('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef double r = float(x)
     return r
@@ -123,6 +150,9 @@
     4
     >>> double_to_py_int(4)
     4
+    >>> double_to_py_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     return int(x)
 
@@ -134,6 +164,9 @@
     4.0
     >>> double_to_double_int(4)
     4.0
+    >>> double_to_double_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef double r = int(x)
     return r
@@ -150,6 +183,9 @@
     4.0
     >>> float_to_float_int(4)
     4.0
+    >>> float_to_float_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef float r = int(x)
     return r
@@ -166,6 +202,9 @@
     4.0
     >>> float_to_double_int(4)
     4.0
+    >>> float_to_double_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef double r = int(x)
     return r
@@ -182,24 +221,9 @@
     4.0
     >>> double_to_float_int(4)
     4.0
-    """
-    cdef float r = int(x)
-    return r
-
-
-@cython.test_fail_if_path_exists("//SingleAssignmentNode//TypecastNode")
-@cython.test_assert_path_exists(
-    "//PythonCapiCallNode",
-    "//PythonCapiCallNode/PythonCapiFunctionNode/@cname = '__Pyx_truncl'",
-)
-def long_double_to_float_int(long double x):
-    """
-    >>> long_double_to_float_int(4.1)
-    4.0
-    >>> long_double_to_float_int(-4.1)
-    -4.0
-    >>> long_double_to_float_int(4)
-    4.0
+    >>> double_to_float_int('4')  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
     """
     cdef float r = int(x)
     return r
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ipow_crash_T562.pyx cython-0.20.1+1~202203241016-9537/tests/run/ipow_crash_T562.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ipow_crash_T562.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ipow_crash_T562.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 562
+# ticket: t562
 
 class IPOW:
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/isnot.pyx cython-0.20.1+1~202203241016-9537/tests/run/isnot.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/isnot.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/isnot.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,12 +3,17 @@
 
 cimport cython
 
+# Use a single global object for identity checks.
+# PyPy can optimise away integer objects, for example, and may fail the 'is' test.
+obj = object()
+
+
 @cython.test_fail_if_path_exists('//NotNode')
 def is_not(a, b):
     """
     >>> is_not(1, 2)
     True
-    >>> x = 1
+    >>> x = obj
     >>> is_not(x, x)
     False
     """
@@ -20,7 +25,7 @@
     """
     >>> not_is_not(1, 2)
     False
-    >>> x = 1
+    >>> x = obj
     >>> not_is_not(x, x)
     True
     """
@@ -32,7 +37,7 @@
     """
     >>> not_is(1, 2)
     True
-    >>> x = 1
+    >>> x = obj
     >>> not_is(x, x)
     False
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/iterdict.pyx cython-0.20.1+1~202203241016-9537/tests/run/iterdict.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/iterdict.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/iterdict.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -555,3 +555,19 @@
     for k, v in dict(*args, **kwargs).iteritems():
         result.append((k, v))
     return result
+
+
+cdef class NotADict:
+    """
+    >>> NotADict().listvalues()  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ...
+    TypeError: descriptor 'values' for 'mappingproxy' objects doesn't apply to a 'iterdict.NotADict' object
+    """
+    cdef long v
+    def __cinit__(self):
+        self.v = 1
+    itervalues = type(object.__dict__).values
+
+    def listvalues(self):
+        return [v for v in self.itervalues()]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/knuth_man_or_boy_test.pyx cython-0.20.1+1~202203241016-9537/tests/run/knuth_man_or_boy_test.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/knuth_man_or_boy_test.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/knuth_man_or_boy_test.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -46,9 +46,13 @@
 def a(in_k, x1, x2, x3, x4, x5):
     """
     >>> import sys
-    >>> sys.setrecursionlimit(1350)
+    >>> old_limit = sys.getrecursionlimit()
+    >>> sys.setrecursionlimit(1350 if not getattr(sys, 'pypy_version_info', None) else 2700)
+
     >>> a(10, 1, -1, -1, 1, 0)
     -67
+
+    >>> sys.setrecursionlimit(old_limit)
     """
     k = [in_k]
     def b():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/kwargproblems.pyx cython-0.20.1+1~202203241016-9537/tests/run/kwargproblems.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/kwargproblems.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/kwargproblems.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -4,7 +4,7 @@
     >>> d = {1 : 2}
     >>> test(**d)       # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     >>> d
     {1: 2}
     >>> d = {}
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/kwargs_passthrough.pyx cython-0.20.1+1~202203241016-9537/tests/run/kwargs_passthrough.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/kwargs_passthrough.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/kwargs_passthrough.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,7 +1,6 @@
-cimport cython
+import cython
 
-
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def wrap_passthrough(f):
     """
     >>> def f(a=1): return a
@@ -80,7 +79,7 @@
     return wrapper
 
 
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def wrap_passthrough2(f):
     """
     >>> def f(a=1): return a
@@ -99,7 +98,7 @@
     return wrapper
 
 
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def wrap_modify(f):
     """
     >>> def f(a=1, test=2):
@@ -123,7 +122,7 @@
     return wrapper
 
 
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def wrap_modify_mix(f):
     """
     >>> def f(a=1, test=2):
@@ -139,6 +138,18 @@
     >>> wrapped(a=2, test=3)
     CALLED
     (2, 1)
+
+    >>> def py_modify(**kwargs):
+    ...     print(sorted(kwargs.items()))
+    ...     kwargs['new'] = len(kwargs)
+    ...     return kwargs
+
+    >>> wrapped_modify = wrap_modify_mix(py_modify)
+    >>> sorted(wrapped_modify(a=1).items())
+    CALLED
+    [('a', 1)]
+    [('a', 1), ('test', 1)]
+    [('a', 1), ('new', 2), ('test', 1)]
     """
     def wrapper(*args, **kwargs):
         print("CALLED")
@@ -175,7 +186,21 @@
     return wrapper
 
 
-@cython.test_assert_path_exists('//MergedDictNode')
+def modify_in_function():
+    """
+    >>> modify_in_function()
+    {'foo': 'bar'}
+    {'foo': 'bar'}
+    """
+    def inner(**kwds):
+        kwds['foo'] = 'modified'
+    d = {'foo': 'bar'}
+    print(d)
+    inner(**d)
+    print(d)
+
+
+#@cython.test_assert_path_exists('//MergedDictNode')
 def wrap_modify_func_mix(f):
     """
     >>> def f(a=1, test=2):
@@ -203,12 +228,11 @@
     return wrapper
 
 
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def wrap_reassign(f):
     """
     >>> def f(a=1, test=2):
     ...     return a, test
-
     >>> wrapped = wrap_reassign(f)
     >>> wrapped(1)
     CALLED
@@ -227,7 +251,7 @@
     return wrapper
 
 
-@cython.test_fail_if_path_exists('//MergedDictNode')
+#@cython.test_fail_if_path_exists('//MergedDictNode')
 def kwargs_metaclass(**kwargs):
     """
     >>> K = kwargs_metaclass()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/lambda_class_T605.pyx cython-0.20.1+1~202203241016-9537/tests/run/lambda_class_T605.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/lambda_class_T605.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/lambda_class_T605.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: lambda
-# ticket: 605
+# ticket: t605
 
 cdef int cdef_CONST = 123
 CONST = 456
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/lambda_module_T603.pyx cython-0.20.1+1~202203241016-9537/tests/run/lambda_module_T603.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/lambda_module_T603.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/lambda_module_T603.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: lambda
-# ticket: 603
+# ticket: t603
 
 # Module scope lambda functions
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/lambda_T195.pyx cython-0.20.1+1~202203241016-9537/tests/run/lambda_T195.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/lambda_T195.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/lambda_T195.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: lambda
-# ticket: 195
+# ticket: t195
 
 __doc__ = u"""
 #>>> py_identity = lambda x:x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/lambda_T723.pyx cython-0.20.1+1~202203241016-9537/tests/run/lambda_T723.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/lambda_T723.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/lambda_T723.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 723
+# ticket: t723
 # tag: lambda
 
 def t723(a):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/language_level.srctree cython-0.20.1+1~202203241016-9537/tests/run/language_level.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/language_level.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/language_level.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -7,12 +7,19 @@
 from Cython.Build.Dependencies import cythonize
 from distutils.core import setup
 
-setup(
-    ext_modules = (cythonize("infile*.py") +
-                   cythonize("directive2.py", compiler_directives={'language_level': 2}) +
-                   cythonize("directive3.py", compiler_directives={'language_level': 3})
-                   )
-)
+ext_modules = []
+
+# Test language_level specified in the cythonize() call
+ext_modules += cythonize("directive2.py", compiler_directives={'language_level': 2})
+ext_modules += cythonize("directive3.py", compiler_directives={'language_level': 3})
+
+# Test language_level specified in the source file. We give a
+# conflicting directive to cythonize() to check that the language_level
+# is correctly overridden when compiling
+ext_modules += cythonize("infile2.py", compiler_directives={'language_level': 3})
+ext_modules += cythonize("infile3.py", compiler_directives={'language_level': 2})
+
+setup(ext_modules=ext_modules)
 
 ######## directive3.py ########
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/large_consts_T237.pyx cython-0.20.1+1~202203241016-9537/tests/run/large_consts_T237.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/large_consts_T237.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/large_consts_T237.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 237
+# ticket: t237
 #def add_large_c():
 #    cdef unsigned long long val = 2**30 + 2**30
 #    return val
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/letnode_T766.pyx cython-0.20.1+1~202203241016-9537/tests/run/letnode_T766.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/letnode_T766.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/letnode_T766.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 766
+# ticket: t766
 # tag: letnode
 
 def test_letnode_range(int n):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/libc_math.pyx cython-0.20.1+1~202203241016-9537/tests/run/libc_math.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/libc_math.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/libc_math.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,8 +2,8 @@
 
 from libc.math cimport (M_E, M_LOG2E, M_LOG10E, M_LN2, M_LN10, M_PI, M_PI_2,
         M_PI_4, M_1_PI, M_2_PI, M_2_SQRTPI, M_SQRT2, M_SQRT1_2)
-from libc.math cimport (acos, asin, atan, atan2, cos, sin, tan, cosh, sinh,
-        tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt)
+from libc.math cimport (acos, asin, atan, atan2, cos, modf, sin, sinf, sinl,
+        tan, cosh, sinh, tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt)
 cimport libc.math as libc_math
 
 
@@ -34,3 +34,21 @@
     [True, True, True, True, True, True, True, True, True, True]
     """
     return sin(x)
+
+
+def test_sin_kwarg(x):
+    """
+    >>> test_sin_kwarg(0)
+    0.0
+    """
+    return sin(x=x)
+
+
+def test_modf(x):
+    """
+    >>> test_modf(2.5)
+    (0.5, 2.0)
+    """
+    cdef double i
+    cdef double f = modf(x, &i)
+    return (f, i)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/libcpp_algo.pyx cython-0.20.1+1~202203241016-9537/tests/run/libcpp_algo.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/libcpp_algo.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/libcpp_algo.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,12 +1,13 @@
+# mode: run
 # tag: cpp
 
 from libcpp cimport bool
-from libcpp.algorithm cimport make_heap, sort_heap, sort, partial_sort
+from libcpp.algorithm cimport make_heap, sort_heap
 from libcpp.vector cimport vector
 
 
 # XXX should use std::greater, but I don't know how to wrap that.
-cdef inline bool greater(int x, int y):
+cdef inline bool greater(const int &x, const int &y):
     return x > y
 
 
@@ -20,40 +21,10 @@
     cdef vector[int] v = l
 
     if reverse:
-        make_heap(v.begin(), v.end(), greater)
-        sort_heap(v.begin(), v.end(), greater)
+        make_heap(v.begin(), v.end(), &greater)
+        sort_heap(v.begin(), v.end(), &greater)
     else:
         make_heap(v.begin(), v.end())
         sort_heap(v.begin(), v.end())
 
     return v
-
-
-def partialsort(l, int k, reverse=False):
-    """
-    >>> partialsort([4, 2, 3, 1, 5], k=2)[:2]
-    [1, 2]
-    >>> partialsort([4, 2, 3, 1, 5], k=2, reverse=True)[:2]
-    [5, 4]
-    """
-    cdef vector[int] v = l
-    if reverse:
-        partial_sort(v.begin(), v.begin() + k, v.end(), greater)
-    else:
-        partial_sort(v.begin(), v.begin() + k, v.end())
-    return v
-
-
-def stdsort(l, reverse=False):
-    """
-    >>> stdsort([3, 2, 1, 4, 5])
-    [1, 2, 3, 4, 5]
-    >>> stdsort([3, 2, 1, 4, 5], reverse=True)
-    [5, 4, 3, 2, 1]
-    """
-    cdef vector[int] v = l
-    if reverse:
-        sort(v.begin(), v.end(), greater)
-    else:
-        sort(v.begin(), v.end())
-    return v
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/libcpp_all.pyx cython-0.20.1+1~202203241016-9537/tests/run/libcpp_all.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/libcpp_all.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/libcpp_all.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,9 +1,10 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 import cython
 
 cimport libcpp
 
+# cimport libcpp.atomic
 cimport libcpp.deque
 cimport libcpp.list
 cimport libcpp.map
@@ -15,6 +16,7 @@
 cimport libcpp.complex
 cimport libcpp.limits
 
+# from libcpp.atomic cimport *
 from libcpp.deque  cimport *
 from libcpp.list   cimport *
 from libcpp.map    cimport *
@@ -26,6 +28,7 @@
 from libcpp.complex cimport *
 from libcpp.limits cimport *
 
+# cdef libcpp.atomic.atomc[int]  a1 = atomic[int]()
 cdef libcpp.deque.deque[int]   d1 = deque[int]()
 cdef libcpp.list.list[int]     l1 = list[int]()
 cdef libcpp.map.map[int,int]   m1 = map[int,int]()
@@ -94,6 +97,7 @@
         cython.operator.preincrement(iter)
     return lst
 
+
 cdef double dmax = numeric_limits[double].max()
 cdef double dmin = numeric_limits[double].min()
 cdef double deps = numeric_limits[double].epsilon()
@@ -124,4 +128,23 @@
     """
     cdef vector[int] x = I
 
-    
+
+def complex_operators():
+    """
+    >>> complex_operators()
+    [-1.0, 0.0, 0.0, 2.0, 0.0, 2.0]
+    """
+    cdef libcpp.complex.complex[double] a = libcpp.complex.complex[double](0.0,1.0)
+    cdef libcpp.complex.complex[double] r1=a*a
+    cdef libcpp.complex.complex[double] r2=a*2.0
+    cdef libcpp.complex.complex[double] r3=2.0*a
+    return [r1.real(), r1.imag(), r2.real(), r2.imag(), r3.real(), r3.imag()]
+
+def pair_comparison():
+    """
+    >>> pair_comparison()
+    [False, True, False, True, False]
+    """
+    cdef pair[double, double] p1 = pair[double, double](1.0,2.0)
+    cdef pair[double, double] p2 = pair[double, double](2.0,2.0)
+    return [p1==p2,p1==p1,p1>p2,p1p2]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/line_profile_test.srctree cython-0.20.1+1~202203241016-9537/tests/run/line_profile_test.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/line_profile_test.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/line_profile_test.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -49,6 +49,18 @@
 profile.runcall(func, 19)
 assert_stats(profile, func.__name__)
 
+from collatz import run_generator, cy_generator
+func = cy_generator
+profile = line_profiler.LineProfiler(func)
+profile.runcall(run_generator, 19)
+assert_stats(profile, func.__name__)
+
+from collatz import run_coro, cy_coro
+func = cy_coro
+profile = line_profiler.LineProfiler(func)
+profile.runcall(run_coro, 19)
+assert_stats(profile, func.__name__)
+
 from collatz import PyClass
 obj = PyClass()
 func = obj.py_pymethod
@@ -93,6 +105,42 @@
 
 
 @cython.binding(True)
+def cy_generator(int n):
+   x = 1
+   for i in range(n):
+       yield x + 2
+       # waste some time to avoid 0 runtimes (line profiler cannot handle those)
+       while (i + x) < n + 10:
+           i += 2
+
+
+@cython.binding(True)
+def run_generator(n):
+    assert len(list(cy_generator(n))) == n
+
+
+@cython.binding(True)
+async def cy_coro(int n):
+    while n > 1:
+        if n % 2 == 0:
+            n //= 2
+        else:
+            n = 3*n+1
+    x =  n  # make sure the computation does not get discarded
+
+
+@cython.binding(True)
+def run_coro(n):
+    coro = cy_coro(n)
+    try:
+        coro.send(None)
+    except StopIteration:
+        assert True
+    else:
+        assert False, "Coroutine did not raise"
+
+
+@cython.binding(True)
 class PyClass(object):
     def py_pymethod(self):
         x = 1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/line_trace.pyx cython-0.20.1+1~202203241016-9537/tests/run/line_trace.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/line_trace.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/line_trace.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,11 +3,13 @@
 # mode: run
 # tag: trace
 
+import sys
+
+from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
+
 cdef extern from "frameobject.h":
     ctypedef struct PyFrameObject:
-        pass
-
-from cpython.ref cimport PyObject
+        PyObject *f_trace
 
 from cpython.pystate cimport (
     Py_tracefunc,
@@ -15,13 +17,13 @@
     PyTrace_C_CALL, PyTrace_C_EXCEPTION, PyTrace_C_RETURN)
 
 cdef extern from *:
-    void PyEval_SetProfile(Py_tracefunc cfunc, object obj)
-    void PyEval_SetTrace(Py_tracefunc cfunc, object obj)
+    void PyEval_SetProfile(Py_tracefunc cfunc, PyObject *obj)
+    void PyEval_SetTrace(Py_tracefunc cfunc, PyObject *obj)
 
 
 map_trace_types = {
     PyTrace_CALL:        'call',
-    PyTrace_EXCEPTION:   'exc',
+    PyTrace_EXCEPTION:   'exception',
     PyTrace_LINE:        'line',
     PyTrace_RETURN:      'return',
     PyTrace_C_CALL:      'ccall',
@@ -30,25 +32,107 @@
 }.get
 
 
-cdef int _trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* arg) except -1:
-    frame, traceobj = _frame, _traceobj
-    traceobj.append((map_trace_types(what), frame.f_lineno - frame.f_code.co_firstlineno))
+cdef int trace_trampoline(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* _arg) except -1:
+    """
+    This is (more or less) what CPython does in sysmodule.c, function trace_trampoline().
+    """
+    cdef PyObject *tmp
+
+    if what == PyTrace_CALL:
+        if _traceobj is NULL:
+            return 0
+        callback = _traceobj
+    elif _frame.f_trace:
+        callback = _frame.f_trace
+    else:
+        return 0
+
+    frame = _frame
+    arg = _arg if _arg else None
+
+    try:
+        result = callback(frame, what, arg)
+    except:
+        PyEval_SetTrace(NULL, NULL)
+        tmp = _frame.f_trace
+        _frame.f_trace = NULL
+        Py_XDECREF(tmp)
+        raise
+
+    if result is not None:
+        # A bug in Py2.6 prevents us from calling the Python-level setter here,
+        # or otherwise we would get miscalculated line numbers. Was fixed in Py2.7.
+        tmp = _frame.f_trace
+        Py_INCREF(result)
+        _frame.f_trace = result
+        Py_XDECREF(tmp)
+
     return 0
 
 
-cdef int _failing_call_trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* arg) except -1:
-    if what == PyTrace_CALL:
-        raise ValueError("failing call trace!")
-    return _trace_func(_traceobj, _frame, what, arg)
+def _create_trace_func(trace):
+    local_names = {}
+
+    def _trace_func(frame, event, arg):
+        if sys.version_info < (3,) and 'line_trace' not in frame.f_code.co_filename:
+            # Prevent tracing into Py2 doctest functions.
+            return None
+
+        trace.append((map_trace_types(event, event), frame.f_lineno - frame.f_code.co_firstlineno))
+
+        lnames = frame.f_code.co_varnames
+        if frame.f_code.co_name in local_names:
+            assert lnames == local_names[frame.f_code.co_name]
+        else:
+            local_names[frame.f_code.co_name] = lnames
+
+        # Currently, the locals dict is empty for Cython code, but not for Python code.
+        if frame.f_code.co_name.startswith('py_'):
+            # Change this when we start providing proper access to locals.
+            assert frame.f_locals, frame.f_code.co_name
+        else:
+            assert not frame.f_locals, frame.f_code.co_name
+
+        return _trace_func
+    return _trace_func
+
+
+def _create_failing_call_trace_func(trace):
+    func = _create_trace_func(trace)
+    def _trace_func(frame, event, arg):
+        if event == PyTrace_CALL:
+            raise ValueError("failing call trace!")
+
+        func(frame, event, arg)
+        return _trace_func
+
+    return _trace_func
 
 
-cdef int _failing_line_trace_func(PyObject* _traceobj, PyFrameObject* _frame, int what, PyObject* arg) except -1:
-    if what == PyTrace_LINE and _traceobj:
-        frame, traceobj = _frame, _traceobj
-        if traceobj and traceobj[0] == frame.f_code.co_name:
-            # first line in the right function => fail!
-            raise ValueError("failing line trace!")
-    return _trace_func(_traceobj, _frame, what, arg)
+def _create__failing_line_trace_func(trace):
+    func = _create_trace_func(trace)
+    def _trace_func(frame, event, arg):
+        if event == PyTrace_LINE and trace:
+            if trace and trace[0] == frame.f_code.co_name:
+                # first line in the right function => fail!
+                raise ValueError("failing line trace!")
+
+        func(frame, event, arg)
+        return _trace_func
+    return _trace_func
+
+
+def _create_disable_tracing(trace):
+    func = _create_trace_func(trace)
+    def _trace_func(frame, event, arg):
+        if frame.f_lineno - frame.f_code.co_firstlineno == 2:
+            PyEval_SetTrace(NULL, NULL)
+            return None
+
+        func(frame, event, arg)
+        return _trace_func
+
+    return _trace_func
 
 
 def cy_add(a,b):
@@ -64,20 +148,46 @@
     return z                     # 5
 
 
+def global_name(global_name):
+    # See GH #1836: accessing "frame.f_locals" deletes locals from globals dict.
+    return global_name + 321
+
+
 cdef int cy_add_nogil(int a, int b) nogil except -1:
     x = a + b   # 1
     return x    # 2
 
 
-def run_trace(func, *args):
+def cy_try_except(func):
+    try:
+        return func()
+    except KeyError as exc:
+        raise AttributeError(exc.args[0])
+
+
+def run_trace(func, *args, bint with_sys=False):
     """
     >>> def py_add(a,b):
     ...     x = a+b
     ...     return x
+
+    >>> def py_add_with_nogil(a,b):
+    ...     x=a; y=b                     # 1
+    ...     for _ in range(1):           # 2
+    ...         z = 0                    # 3
+    ...         z += py_add(x, y)        # 4
+    ...     return z                     # 5
+
     >>> run_trace(py_add, 1, 2)
     [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
     >>> run_trace(cy_add, 1, 2)
     [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+
+    >>> run_trace(py_add, 1, 2, with_sys=True)
+    [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> run_trace(cy_add, 1, 2, with_sys=True)
+    [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+
     >>> result = run_trace(cy_add_with_nogil, 1, 2)
     >>> result[:5]
     [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4)]
@@ -85,37 +195,142 @@
     [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
     >>> result[9:]
     [('line', 2), ('line', 5), ('return', 5)]
+
+    >>> result = run_trace(cy_add_with_nogil, 1, 2, with_sys=True)
+    >>> result[:5]  # sys
+    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4)]
+    >>> result[5:9]  # sys
+    [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> result[9:]  # sys
+    [('line', 2), ('line', 5), ('return', 5)]
+
+    >>> result = run_trace(py_add_with_nogil, 1, 2)
+    >>> result[:5]  # py
+    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4)]
+    >>> result[5:9]  # py
+    [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> result[9:]  # py
+    [('line', 2), ('line', 5), ('return', 5)]
+
+    >>> run_trace(global_name, 123)
+    [('call', 0), ('line', 2), ('return', 2)]
+    >>> run_trace(global_name, 111)
+    [('call', 0), ('line', 2), ('return', 2)]
+    >>> run_trace(global_name, 111, with_sys=True)
+    [('call', 0), ('line', 2), ('return', 2)]
+    >>> run_trace(global_name, 111, with_sys=True)
+    [('call', 0), ('line', 2), ('return', 2)]
     """
     trace = []
-    PyEval_SetTrace(_trace_func, trace)
+    trace_func = _create_trace_func(trace)
+    if with_sys:
+        sys.settrace(trace_func)
+    else:
+        PyEval_SetTrace(trace_trampoline, trace_func)
     try:
         func(*args)
     finally:
-        PyEval_SetTrace(NULL, None)
+        if with_sys:
+            sys.settrace(None)
+        else:
+            PyEval_SetTrace(NULL, NULL)
     return trace
 
 
+def run_trace_with_exception(func, bint with_sys=False, bint fail=False):
+    """
+    >>> def py_return(retval=123): return retval
+    >>> run_trace_with_exception(py_return)
+    OK: 123
+    [('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('return', 0), ('return', 2)]
+    >>> run_trace_with_exception(py_return, with_sys=True)
+    OK: 123
+    [('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('return', 0), ('return', 2)]
+
+    >>> run_trace_with_exception(py_return, fail=True)
+    ValueError('failing line trace!')
+    [('call', 0)]
+
+    #>>> run_trace_with_exception(lambda: 123, with_sys=True, fail=True)
+    #ValueError('huhu')
+    #[('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('return', 0), ('return', 2)]
+
+    >>> def py_raise_exc(exc=KeyError('huhu')): raise exc
+    >>> run_trace_with_exception(py_raise_exc)
+    AttributeError('huhu')
+    [('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('exception', 0), ('return', 0), ('line', 3), ('line', 4), ('return', 4)]
+    >>> run_trace_with_exception(py_raise_exc, with_sys=True)
+    AttributeError('huhu')
+    [('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('exception', 0), ('return', 0), ('line', 3), ('line', 4), ('return', 4)]
+    >>> run_trace_with_exception(py_raise_exc, fail=True)
+    ValueError('failing line trace!')
+    [('call', 0)]
+
+    #>>> run_trace_with_exception(raise_exc, with_sys=True, fail=True)
+    #ValueError('huhu')
+    #[('call', 0), ('line', 1), ('line', 2), ('call', 0), ('line', 0), ('exception', 0), ('return', 0), ('line', 3), ('line', 4), ('return', 4)]
+    """
+    trace = ['cy_try_except' if fail else 'NO ERROR']
+    trace_func = _create__failing_line_trace_func(trace) if fail else _create_trace_func(trace)
+    if with_sys:
+        sys.settrace(trace_func)
+    else:
+        PyEval_SetTrace(trace_trampoline, trace_func)
+    try:
+        try:
+            retval = cy_try_except(func)
+        except ValueError as exc:
+            print("%s(%r)" % (type(exc).__name__, str(exc)))
+        except AttributeError as exc:
+            print("%s(%r)" % (type(exc).__name__, str(exc)))
+        else:
+            print('OK: %r' % retval)
+    finally:
+        if with_sys:
+            sys.settrace(None)
+        else:
+            PyEval_SetTrace(NULL, NULL)
+    return trace[1:]
+
+
 def fail_on_call_trace(func, *args):
     """
     >>> def py_add(a,b):
     ...     x = a+b
     ...     return x
+
     >>> fail_on_call_trace(py_add, 1, 2)
     Traceback (most recent call last):
     ValueError: failing call trace!
+
+    >>> fail_on_call_trace(cy_add, 1, 2)
+    Traceback (most recent call last):
+    ValueError: failing call trace!
     """
     trace = []
-    PyEval_SetTrace(_failing_call_trace_func, trace)
+    trace_func = _create_failing_call_trace_func(trace)
+    PyEval_SetTrace(trace_trampoline, trace_func)
     try:
         func(*args)
     finally:
-        PyEval_SetTrace(NULL, None)
+        PyEval_SetTrace(NULL, NULL)
     assert not trace
 
 
-def fail_on_line_trace(fail_func):
+def fail_on_line_trace(fail_func, add_func, nogil_add_func):
     """
-    >>> result = fail_on_line_trace(None)
+    >>> def py_add(a,b):
+    ...     x = a+b       # 1
+    ...     return x      # 2
+
+    >>> def py_add_with_nogil(a,b):
+    ...     x=a; y=b                     # 1
+    ...     for _ in range(1):           # 2
+    ...         z = 0                    # 3
+    ...         z += py_add(x, y)        # 4
+    ...     return z                     # 5
+
+    >>> result = fail_on_line_trace(None, cy_add, cy_add_with_nogil)
     >>> len(result)
     17
     >>> result[:5]
@@ -127,37 +342,98 @@
     >>> result[14:]
     [('line', 2), ('line', 5), ('return', 5)]
 
-    >>> result = fail_on_line_trace('cy_add_with_nogil')
+    >>> result = fail_on_line_trace(None, py_add, py_add_with_nogil)
+    >>> len(result)
+    17
+    >>> result[:5]  # py
+    ['NO ERROR', ('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> result[5:10]  # py
+    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4)]
+    >>> result[10:14]  # py
+    [('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> result[14:]  # py
+    [('line', 2), ('line', 5), ('return', 5)]
+
+    >>> result = fail_on_line_trace('cy_add_with_nogil', cy_add, cy_add_with_nogil)
     failing line trace!
     >>> result
-    ['cy_add_with_nogil', ('call', 0), ('line', 1), ('line', 2), ('return', 2), ('call', 0), ('return', 1)]
+    ['cy_add_with_nogil', ('call', 0), ('line', 1), ('line', 2), ('return', 2), ('call', 0)]
+
+    >>> result = fail_on_line_trace('py_add_with_nogil', py_add, py_add_with_nogil)  # py
+    failing line trace!
+    >>> result  # py
+    ['py_add_with_nogil', ('call', 0), ('line', 1), ('line', 2), ('return', 2), ('call', 0)]
 
-    >>> result = fail_on_line_trace('cy_add_nogil')
+    >>> result = fail_on_line_trace('cy_add_nogil', cy_add, cy_add_with_nogil)
     failing line trace!
     >>> result[:5]
     ['cy_add_nogil', ('call', 0), ('line', 1), ('line', 2), ('return', 2)]
     >>> result[5:]
-    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4), ('call', 0), ('return', 1), ('return', 4)]
+    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4), ('call', 0)]
+
+    >>> result = fail_on_line_trace('py_add', py_add, py_add_with_nogil)  # py
+    failing line trace!
+    >>> result[:5]  # py
+    ['py_add', ('call', 0), ('line', 1), ('line', 2), ('return', 2)]
+    >>> result[5:]  # py
+    [('call', 0), ('line', 1), ('line', 2), ('line', 3), ('line', 4), ('call', 0)]
     """
     cdef int x = 1
     trace = ['NO ERROR']
     exception = None
-    PyEval_SetTrace(_failing_line_trace_func, trace)
+    trace_func = _create__failing_line_trace_func(trace)
+    PyEval_SetTrace(trace_trampoline, trace_func)
     try:
         x += 1
-        cy_add(1, 2)
+        add_func(1, 2)
         x += 1
         if fail_func:
             trace[0] = fail_func  # trigger error on first line
         x += 1
-        cy_add_with_nogil(3, 4)
+        nogil_add_func(3, 4)
         x += 1
     except Exception as exc:
         exception = str(exc)
     finally:
-        PyEval_SetTrace(NULL, None)
+        PyEval_SetTrace(NULL, NULL)
     if exception:
         print(exception)
     else:
         assert x == 5
     return trace
+
+
+def disable_trace(func, *args, bint with_sys=False):
+    """
+    >>> def py_add(a,b):
+    ...     x = a+b
+    ...     return x
+    >>> disable_trace(py_add, 1, 2)
+    [('call', 0), ('line', 1)]
+    >>> disable_trace(py_add, 1, 2, with_sys=True)
+    [('call', 0), ('line', 1)]
+
+    >>> disable_trace(cy_add, 1, 2)
+    [('call', 0), ('line', 1)]
+    >>> disable_trace(cy_add, 1, 2, with_sys=True)
+    [('call', 0), ('line', 1)]
+
+    >>> disable_trace(cy_add_with_nogil, 1, 2)
+    [('call', 0), ('line', 1)]
+    >>> disable_trace(cy_add_with_nogil, 1, 2, with_sys=True)
+    [('call', 0), ('line', 1)]
+    """
+    trace = []
+    trace_func = _create_disable_tracing(trace)
+    if with_sys:
+        sys.settrace(trace_func)
+    else:
+        PyEval_SetTrace(trace_trampoline, trace_func)
+    try:
+        func(*args)
+    finally:
+        if with_sys:
+            sys.settrace(None)
+        else:
+            PyEval_SetTrace(NULL, NULL)
+    return trace
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/list_comp_in_closure_T598.pyx cython-0.20.1+1~202203241016-9537/tests/run/list_comp_in_closure_T598.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/list_comp_in_closure_T598.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/list_comp_in_closure_T598.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: closures
-# ticket: 598
+# ticket: t598
 # cython: language_level=3
 
 def list_comp_in_closure():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/listcomp.pyx cython-0.20.1+1~202203241016-9537/tests/run/listcomp.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/listcomp.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/listcomp.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,9 @@
+# mode: run
+# tag: listcomp, comprehension
+
 cimport cython
 
+
 def smoketest():
     """
     >>> smoketest()
@@ -10,6 +14,7 @@
     assert x != 'abc'
     return result
 
+
 def list_genexp():
     """
     >>> list_genexp()
@@ -20,6 +25,7 @@
     assert x == 'abc'
     return result
 
+
 def int_runvar():
     """
     >>> int_runvar()
@@ -28,6 +34,7 @@
     cdef int x
     print [x*2 for x in range(5) if x % 2 == 0]
 
+
 cdef class A:
     def __repr__(self): return u"A"
 
@@ -39,6 +46,7 @@
     cdef A obj
     print [obj for obj in [A(), A(), A()]]
 
+
 def inferred_type():
     """
     >>> inferred_type()
@@ -46,6 +54,7 @@
     """
     print [cython.typeof(obj) for obj in [A(), A(), A()]]
 
+
 def not_inferred_type():
     """
     >>> not_inferred_type()
@@ -53,6 +62,7 @@
     """
     print [cython.typeof(obj) for obj in [1, A(), 'abc']]
 
+
 def iterdict():
     """
     >>> iterdict()
@@ -63,6 +73,7 @@
     l.sort()
     print l
 
+
 listcomp_result = [ i*i for i in range(5) ]
 def global_listcomp():
     """
@@ -72,6 +83,25 @@
     [0, 1, 4, 9, 16]
     """
 
+
+class ListCompInClass(object):
+    """
+    >>> x = ListCompInClass()
+    >>> x.listcomp
+    [1, 2, 3]
+    """
+    listcomp = [i+1 for i in range(3)]
+
+
+cdef class ListCompInCClass:
+    """
+    >>> x = ListCompInCClass()
+    >>> x.listcomp
+    [1, 2, 3]
+    """
+    listcomp = [i+1 for i in range(3)]
+
+
 def nested_result():
     """
     >>> nested_result()
@@ -80,6 +110,7 @@
     result = [[a-1 for a in range(b)] for b in range(4)]
     return result
 
+
 def listcomp_as_condition(sequence):
     """
     >>> listcomp_as_condition(['a', 'b', '+'])
@@ -93,15 +124,21 @@
         return True
     return False
 
+
 @cython.test_fail_if_path_exists("//SimpleCallNode//ComprehensionNode")
 @cython.test_assert_path_exists("//ComprehensionNode")
 def sorted_listcomp(sequence):
     """
+    >>> sorted_listcomp([])
+    []
+    >>> sorted_listcomp([1])
+    [2]
     >>> sorted_listcomp([3,2,4])
     [3, 4, 5]
     """
     return sorted([ n+1 for n in sequence ])
 
+
 @cython.test_fail_if_path_exists("//IfStatNode",
                                  "//ComprehensionAppendNode")
 @cython.test_assert_path_exists("//ComprehensionNode")
@@ -112,6 +149,58 @@
     """
     return [x*2 for x in range(3) if False]
 
+
+@cython.test_fail_if_path_exists("//IfStatNode",
+                                 "//ComprehensionAppendNode")
+@cython.test_assert_path_exists("//ComprehensionNode")
+def listcomp_const_condition_false_bool_test():
+    """
+    >>> listcomp_const_condition_false_bool_test()
+    True
+    """
+    return not [l for l in [1] if False]
+
+
+@cython.test_fail_if_path_exists("//IfStatNode",
+                                 "//ComprehensionAppendNode")
+@cython.test_assert_path_exists("//ComprehensionNode")
+def listcomp_const_condition_false_assert():
+    """
+    >>> listcomp_const_condition_false_assert()
+    """
+    assert not [l for l in [1] if False]
+
+
+@cython.test_fail_if_path_exists("//ComprehensionNode//IfStatNode",
+                                 "//ComprehensionAppendNode")
+@cython.test_assert_path_exists("//ComprehensionNode",
+                                "//IfStatNode")
+def listcomp_const_condition_false_if():
+    """
+    >>> listcomp_const_condition_false_if()
+    True
+    """
+    if not [l for l in [1] if False]:
+        return True
+    return False
+
+
+@cython.test_fail_if_path_exists("//ComprehensionNode//IfStatNode",
+                                 "//ComprehensionAppendNode")
+@cython.test_assert_path_exists("//ComprehensionNode",
+                                "//IfStatNode")
+def listcomp_const_condition_false_typed_error():
+    """
+    >>> listcomp_const_condition_false_typed_error()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    """
+    cdef str l
+    if not [l for l in [1] if False]:
+        return True
+    return False
+
+
 @cython.test_fail_if_path_exists("//IfStatNode")
 @cython.test_assert_path_exists("//ComprehensionNode",
                                 "//ComprehensionAppendNode")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/list_pop.pyx cython-0.20.1+1~202203241016-9537/tests/run/list_pop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/list_pop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/list_pop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -206,7 +206,7 @@
     """
     >>> crazy_pop(list(range(10)))    # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: pop... at most ... argument...
+    TypeError: pop... argument...
     >>> crazy_pop(A())
     (1, 2, 3)
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/list.pyx cython-0.20.1+1~202203241016-9537/tests/run/list.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/list.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/list.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -79,6 +79,10 @@
     l1.reverse()
     return l1
 
+
+@cython.test_assert_path_exists(
+    '//SimpleCallNode//AttributeNode[@entry.cname = "__Pyx_PyList_Append"]',
+)
 def test_list_append():
     """
     >>> test_list_append()
@@ -89,6 +93,36 @@
     l1.append(4)
     return l1
 
+
+@cython.test_assert_path_exists(
+    '//SimpleCallNode//NameNode[@entry.cname = "__Pyx_PyList_Append"]',
+)
+def test_list_append_unbound():
+    """
+    >>> test_list_append_unbound()
+    [1, 2, 3, 4]
+    """
+    cdef list l1 = [1,2]
+    list.append(l1, 3)
+    list.append(l1, 4)
+    return l1
+
+
+@cython.test_assert_path_exists(
+    '//SimpleCallNode//NameNode[@entry.cname = "__Pyx_PyList_Append"]',
+)
+def test_list_append_unbound_assigned():
+    """
+    >>> test_list_append_unbound_assigned()
+    [1, 2, 3, 4]
+    """
+    append = list.append
+    cdef list l1 = [1,2]
+    append(l1, 3)
+    append(l1, 4)
+    return l1
+
+
 def test_list_append_insert():
     """
     >>> test_list_append_insert()
@@ -138,20 +172,127 @@
         return i == 2
     return False
 
-def test_list_extend():
+
+@cython.test_assert_path_exists(
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]',
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]',
+)
+def test_list_extend(seq=None, x=4):
     """
     >>> test_list_extend()
-    [1, 2, 3, 4, 5, 6]
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
+    >>> test_list_extend([])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
+    >>> test_list_extend([1])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1]
+    >>> test_list_extend([1, 2])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2]
     """
     cdef list l = [1,2,3]
     l.extend([])
     l.extend(())
-    l.extend(set())
+    l.extend(set())  # not currently optimised (not worth the trouble)
     assert l == [1,2,3]
     assert len(l) == 3
-    l.extend([4,5,6])
+    l.extend([4,x+1,6])
+    l.extend([7,8,9,10,11,12,13,14,15,16])
+    if seq is not None:
+        l.extend(seq)
     return l
 
+
+@cython.test_assert_path_exists(
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]',
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]',
+)
+def test_list_extend_unbound(seq=None, x=4):
+    """
+    >>> test_list_extend_unbound()
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
+    >>> test_list_extend_unbound([])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
+    >>> test_list_extend_unbound([1])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1]
+    >>> test_list_extend_unbound([1, 2])
+    [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2]
+    """
+    cdef list l = [1,2,3]
+    list.extend(l, [])
+    list.extend(l, ())
+    try:
+        list.extend((), ())
+    except TypeError:
+        pass
+    else:
+        assert False, "TypeError not raised!"
+    list.extend(l, set())  # not currently optimised (not worth the trouble)
+    assert l == [1,2,3]
+    assert len(l) == 3
+    list.extend(l, [4,x+1,6])
+    list.extend(l, [7,8,9,10,11,12,13,14,15,16])
+    if seq is not None:
+        list.extend(l, seq)
+    return l
+
+@cython.test_assert_path_exists(
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_ListComp_Append"]',
+    '//PythonCapiCallNode//PythonCapiFunctionNode[@cname = "__Pyx_PyList_Append"]',
+)
+def test_list_extend_sideeffect(seq=None, exc=False):
+    """
+    >>> test_list_extend_sideeffect()
+    ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [4, 6, 7, 8])
+    >>> test_list_extend_sideeffect([])
+    ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], [4, 6, 7, 8])
+    >>> test_list_extend_sideeffect([], exc=True)
+    ([1, 2, 3, 10, 11, 12, 13, 14, 15, 16], [4, 7, 8])
+    >>> test_list_extend_sideeffect([1])
+    ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1], [4, 6, 7, 8])
+    >>> test_list_extend_sideeffect([1], exc=True)
+    ([1, 2, 3, 10, 11, 12, 13, 14, 15, 16, 1], [4, 7, 8])
+    >>> test_list_extend_sideeffect([1, 2])
+    ([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2], [4, 6, 7, 8])
+    """
+    calls = []
+    def sideeffect(value):
+        calls.append(value)
+        return value
+    def fail(value):
+        if exc:
+            raise TypeError("HUHU")
+        return value
+
+    cdef list l = [1,2,3]
+    l.extend([])
+    l.extend(())
+    l.extend(set())  # not currently optimised (not worth the trouble)
+    assert l == [1,2,3]
+    assert len(l) == 3
+
+    # Must first build all items, then append them in order.
+    # If building one value fails, none of them must be appended.
+    try:
+        l.extend([sideeffect(4), fail(5), sideeffect(6)])
+    except TypeError as e:
+        assert exc
+        assert "HUHU" in str(e)
+    else:
+        assert not exc
+
+    try:
+        l.extend([sideeffect(7), sideeffect(8), fail(9)])
+    except TypeError as e:
+        assert exc
+        assert "HUHU" in str(e)
+    else:
+        assert not exc
+
+    l.extend([10,11,12,13,14,15,16])
+    if seq is not None:
+        l.extend(seq)
+    return l, calls
+
+
 def test_none_list_extend(list l):
     """
     >>> test_none_list_extend([])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/locals_expressions_T430.pyx cython-0.20.1+1~202203241016-9537/tests/run/locals_expressions_T430.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/locals_expressions_T430.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/locals_expressions_T430.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 430
+# ticket: t430
 
 __doc__ = u"""
 >>> sorted( get_locals(1,2,3, k=5) .items())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/locals_rebind_T429.pyx cython-0.20.1+1~202203241016-9537/tests/run/locals_rebind_T429.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/locals_rebind_T429.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/locals_rebind_T429.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 429
+# ticket: t429
 
 __doc__ = u"""
 >>> sorted( get_locals(1,2,3, k=5) .items())
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/locals_T732.pyx cython-0.20.1+1~202203241016-9537/tests/run/locals_T732.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/locals_T732.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/locals_T732.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 731
+# ticket: t731
 # tag: locals, vars, dir
 
 cimport cython
@@ -23,8 +23,8 @@
     >>> klass = test_class_locals_and_dir()
     >>> 'visible' in klass.locs and 'not_visible' not in klass.locs
     True
-    >>> klass.names
-    ['__module__', '__qualname__', 'visible']
+    >>> [n for n in klass.names if n not in {"__qualname__", "__annotations__"}]
+    ['__module__', 'visible']
     """
     not_visible = 1234
     class Foo:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/longintrepr.pyx cython-0.20.1+1~202203241016-9537/tests/run/longintrepr.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/longintrepr.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/longintrepr.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,55 @@
+# mode: run
+
+# Test longintrepr declarations by implementing a simple function
+
+from cpython.longintrepr cimport *
+cimport cython
+
+@cython.cdivision(True)
+def lshift(long a, unsigned long n):
+    """
+    Return a * 2^n as Python long.
+
+    >>> print(lshift(3, 1))
+    6
+    >>> print(lshift(1, 30))
+    1073741824
+    >>> print(lshift(-12345, 115))
+    512791237748899576593671817473776680960
+    >>> print(-12345 << 115)
+    -512791237748899576593671817473776680960
+    >>> [i for i in range(100) if (65535 << i) != lshift(65535, i)]
+    []
+    >>> print(lshift(0, 12345))
+    0
+    >>> print(lshift(2**62, 0))   # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    OverflowError...
+    """
+    if not a:
+        return _PyLong_New(0)
+    cdef unsigned long apos = a if a > 0 else -a
+    if (apos >> 1) >= PyLong_BASE:
+        raise OverflowError
+
+    cdef unsigned long index = n // PyLong_SHIFT
+    cdef unsigned long shift = n % PyLong_SHIFT
+
+    cdef digit d = apos
+    cdef digit low = (d << shift) & PyLong_MASK
+    cdef digit high = (d >> (PyLong_SHIFT - shift))
+
+    if high == 0:
+        ret = _PyLong_New(index + 1)
+        ret.ob_digit[index] = low
+    else:
+        ret = _PyLong_New(index + 2)
+        ret.ob_digit[index] = low
+        ret.ob_digit[index + 1] = high
+
+    while index >= 1:
+        index -= 1
+        ret.ob_digit[index] = 0
+
+    return ret
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/lvalue_refs.pyx cython-0.20.1+1~202203241016-9537/tests/run/lvalue_refs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/lvalue_refs.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/lvalue_refs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# tag: cpp
+# tag: cpp, no-cpp-locals
 
 from libcpp.vector cimport vector
 
@@ -15,7 +15,7 @@
 def test_lvalue_ref_assignment():
     cdef vector[dpp]        bar
     cdef vector[vector[dp]] baz
-    cdef vector[double]     data
+    cdef vector[double]     data = [0.0]
     cdef dp                 bongle = &data[0]
 
     bar.resize(1)
@@ -28,3 +28,27 @@
 
     assert bar[0] == &baz[0][0]
     assert bar[0][0] == bongle
+
+cdef void assign_to_basic_reference(int& ref):
+    ref = 123
+
+def test_assign_to_basic_ref():
+    """
+    >>> test_assign_to_basic_ref()
+    123
+    """
+    cdef int x=0
+    assign_to_basic_reference(x)
+    print x
+
+# not *strictly* lvalue refs but this file seems the closest applicable place for it.
+# GH 3754 - std::vector operator[] returns a reference, and this causes problems if
+# the reference is passed into Cython __Pyx_GetItemInt
+def test_ref_used_for_indexing():
+    """
+    >>> test_ref_used_for_indexing()
+    'looked up correctly'
+    """
+    cdef vector[int] idx = [1,2,3]
+    d = {1: "looked up correctly", 2:"oops"}
+    return d[idx[0]]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/matrix_multiplier.pyx cython-0.20.1+1~202203241016-9537/tests/run/matrix_multiplier.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/matrix_multiplier.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/matrix_multiplier.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -22,9 +22,11 @@
 >>> x @ y
 Traceback (most recent call last):
 TypeError: unsupported operand type(s) for @: 'int' and 'int'
->>> x @= y
+
+PyPy exception message has '@' rather than '@='
+>>> x @= y  # doctest: +ELLIPSIS
 Traceback (most recent call last):
-TypeError: unsupported operand type(s) for @=: 'int' and 'int'
+TypeError: unsupported operand type(s) for @...: 'int' and 'int'
 
 >>> y = MatMult(22)
 >>> x @= y
@@ -112,9 +114,9 @@
     >>> print(test_imatmul(MatMult('abc'), 11))
     MatMult("MatMult('abc') @ 11")
 
-    >>> test_imatmul(1, 2)
+    >>> test_imatmul(1, 2)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for @=: 'int' and 'int'
+    TypeError: unsupported operand type(s) for @...: 'int' and 'int'
     """
     a @= b
     return a
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/metaclass.pyx cython-0.20.1+1~202203241016-9537/tests/run/metaclass.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/metaclass.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/metaclass.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -69,8 +69,8 @@
     321
     >>> obj.metaclass_was_here
     True
-    >>> obj._order
-    ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here']
+    >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}]
+    ['__module__', '__doc__', 'bar', 'metaclass_was_here']
     """
     bar = 321
 
@@ -81,8 +81,8 @@
     345
     >>> obj.metaclass_was_here
     True
-    >>> obj._order
-    ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here']
+    >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}]
+    ['__module__', '__doc__', 'bar', 'metaclass_was_here']
     """
     bar = 345
 
@@ -109,8 +109,8 @@
     123
     >>> obj.bar
     321
-    >>> obj._order
-    ['__module__', '__qualname__', '__doc__', 'bar', 'foo']
+    >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}]
+    ['__module__', '__doc__', 'bar', 'foo']
     """
     bar = 321
 
@@ -122,8 +122,8 @@
     567
     >>> obj.bar
     321
-    >>> obj._order
-    ['__module__', '__qualname__', '__doc__', 'bar', 'foo']
+    >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}]
+    ['__module__', '__doc__', 'bar', 'foo']
     """
     bar = 321
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_cdef.pxd cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_cdef.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_cdef.pxd	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_cdef.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,3 @@
+cdef class InPxd:
+    cdef public int __y
+    cdef int __private_cdef(self)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_cdef.pyx cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_cdef.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_cdef.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_cdef.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,95 @@
+# mode: run
+
+def call_cdt_private_cdef(CDefTest o):
+    return o._CDefTest__private_cdef()
+
+cdef __c_func():
+    return "cdef function"
+
+cdef __c_var = "Shouldn't see this"
+
+cdef class CDefTest:
+    """
+    >>> cd = CDefTest()
+    >>> '_CDefTest__private' in dir(cd)
+    True
+    >>> cd._CDefTest__private()
+    8
+    >>> call_cdt_private_cdef(cd)
+    8
+    >>> '__private' in dir(cd)
+    False
+    >>> '_CDefTest__x' in dir(cd)
+    True
+
+    >>> '__x' in dir(cd)
+    False
+    >>> cd._CDefTest__y
+    2
+    """
+    __x = 1
+    cdef public int __y
+
+    def __init__(self):
+        self.__y = 2
+
+    def __private(self): return 8
+
+    cdef __private_cdef(self): return 8
+
+    def get(self):
+        """
+        >>> CDefTest().get()
+        (1, 1, 8)
+        """
+        return self._CDefTest__x, self.__x, self.__private()
+
+    def get_inner(self):
+        """
+        >>> CDefTest().get_inner()
+        (1, 1, 8)
+        """
+        def get(o):
+            return o._CDefTest__x, o.__x, o.__private()
+        return get(self)
+
+    def get_c_func(self):
+        """
+        Should still be able to access C function with __names
+        >>> CDefTest().get_c_func()
+        'cdef function'
+        """
+        return __c_func()
+
+    def get_c_func2(self):
+        """
+        Should find mangled name before C __name
+        >>> CDefTest().get_c_func2()
+        'lambda'
+        """
+        _CDefTest__c_func = lambda: "lambda"
+        return __c_func()
+
+    def get_c_var(self):
+        """
+        >>> CDefTest().get_c_var()
+        'c var'
+        """
+        global __c_var
+        __c_var = "c var"
+        return __c_var
+
+def call_inpdx_private_cdef(InPxd o):
+    return o._InPxd__private_cdef()
+
+cdef class InPxd:
+    """
+    >>> InPxd()._InPxd__y
+    2
+    >>> call_inpdx_private_cdef(InPxd())
+    8
+    """
+    def __init__(self):
+        self.__y = 2
+
+    cdef int __private_cdef(self): return 8
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_pure.py cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_pure.py
--- cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_pure.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_pure.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,76 @@
+# mode: run
+# cython: language_level=3
+
+# This file tests that methodmangling is applied correctly to
+# pure Python decorated classes.
+
+import cython
+
+if cython.compiled:
+    # don't run in Python mode since a significant number of the tests
+    # are only for Cython features
+
+    def declare(**kwargs):
+        return kwargs['__x']
+
+    class RegularClass:
+        @cython.locals(__x=cython.int)
+        def f1(self, __x, dummy=None):
+            """
+            Is the locals decorator correctly applied
+            >>> c = RegularClass()
+            >>> c.f1(1)
+            1
+            >>> c.f1("a")
+            Traceback (most recent call last):
+            ...
+            TypeError: an integer is required
+            >>> c.f1(_RegularClass__x = 1)
+            1
+            """
+            return __x
+
+        def f2(self, x):
+            """
+            Is the locals decorator correctly applied
+            >>> c = RegularClass()
+            >>> c.f2(1)
+            1
+            >>> c.f2("a")
+            Traceback (most recent call last):
+            ...
+            TypeError: an integer is required
+            """
+            __x = cython.declare(cython.int, x)
+
+            return __x
+
+        def f3(self, x):
+            """
+            Is the locals decorator correctly applied
+            >>> c = RegularClass()
+            >>> c.f3(1)
+            1
+            >>> c.f3("a")
+            Traceback (most recent call last):
+            ...
+            TypeError: an integer is required
+            """
+            cython.declare(__x=cython.int)
+            __x = x
+
+            return __x
+
+        def f4(self, x):
+            """
+            We shouldn't be tripped up by a function called
+            "declare" that is nothing to do with cython
+            >>> RegularClass().f4(1)
+            1
+            """
+            return declare(__x=x)
+else:
+    __doc__ = """
+    >>> True
+    True
+    """  # stops Python2 from failing
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_T5.py cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_T5.py
--- cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_T5.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_T5.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,10 @@
 # mode: run
-# ticket: 5
+# ticket: t5
+
+# A small number of extra tests checking:
+# 1) this works correctly with pure-Python-mode decorators - methodmangling_pure.py.
+# 2) this works correctly with cdef classes - methodmangling_cdef.pyx
+# 3) with "error_on_unknown_names" - methodmangling_unknown_names.py
 
 class CyTest(object):
     """
@@ -15,8 +20,23 @@
 
     >>> '__x' in dir(cy)
     False
+    >>> cy._CyTest__y
+    2
+
+    >>> '_CyTest___more_than_two' in dir(cy)
+    True
+    >>> '___more_than_two' in dir(cy)
+    False
+    >>> '___more_than_two_special___' in dir(cy)
+    True
     """
     __x = 1
+    ___more_than_two = 3
+    ___more_than_two_special___ = 4
+
+    def __init__(self):
+        self.__y = 2
+
     def __private(self): return 8
 
     def get(self):
@@ -88,8 +108,285 @@
     1
     >>> ut.get()
     1
+    >>> ut._UnderscoreTest__UnderscoreNested().ret1()
+    1
+    >>> ut._UnderscoreTest__UnderscoreNested.__name__
+    '__UnderscoreNested'
+    >>> ut._UnderscoreTest__prop
+    1
     """
     __x = 1
 
     def get(self):
         return self.__x
+
+    class __UnderscoreNested(object):
+        def ret1(self):
+            return 1
+
+    @property
+    def __prop(self):
+        return self.__x
+
+class C:
+    error = """Traceback (most recent call last):
+...
+TypeError:
+"""
+    __doc__ = """
+>>> instance = C()
+
+Instance methods have their arguments mangled
+>>> instance.method1(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> instance.method1(_C__arg=1)
+1
+>>> instance.method2(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> instance.method2(_C__arg=1)
+1
+
+Works when optional argument isn't passed
+>>> instance.method2()
+None
+
+Where args are in the function's **kwargs dict, names aren't mangled
+>>> instance.method3(__arg=1) # doctest:
+1
+>>> instance.method3(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+Traceback (most recent call last):
+...
+KeyError:
+
+Lambda functions behave in the same way:
+>>> instance.method_lambda(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> instance.method_lambda(_C__arg=1)
+1
+
+Class methods - have their arguments mangled
+>>> instance.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> instance.class_meth(_C__arg=1)
+1
+>>> C.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> C.class_meth(_C__arg=1)
+1
+
+Static methods - have their arguments mangled
+>>> instance.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> instance.static_meth(_C__arg=1)
+1
+>>> C.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+>>> C.static_meth(_C__arg=1)
+1
+
+Functions assigned to the class don't have their arguments mangled
+>>> instance.class_assigned_function(__arg=1)
+1
+>>> instance.class_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+
+Functions assigned to an instance don't have their arguments mangled
+>>> instance.instance_assigned_function = free_function2
+>>> instance.instance_assigned_function(__arg=1)
+1
+>>> instance.instance_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL
+{error}
+
+Locals are reported as mangled
+>>> list(sorted(k for k in instance.get_locals(1).keys()))
+['_C__arg', 'self']
+""".format(error=error)
+
+    def method1(self, __arg):
+        print(__arg)
+
+    def method2(self, __arg=None):
+        # __arg is optional
+        print(__arg)
+
+    def method3(self, **kwargs):
+        print(kwargs['__arg'])
+
+    method_lambda = lambda self, __arg: __arg
+
+    def get_locals(self, __arg):
+        return locals()
+
+    @classmethod
+    def class_meth(cls, __arg):
+        print(__arg)
+
+    @staticmethod
+    def static_meth(__arg, dummy_arg=None):
+        # dummy_arg is to mask https://github.com/cython/cython/issues/3090
+        print(__arg)
+
+def free_function1(x, __arg):
+    print(__arg)
+
+def free_function2(__arg, dummy_arg=None):
+    # dummy_arg is to mask https://github.com/cython/cython/issues/3090
+    print(__arg)
+
+C.class_assigned_function = free_function1
+
+__global_arg = True
+
+_D__arg1 = None
+_D__global_arg = False  # define these because otherwise Cython gives a compile-time error
+       # while Python gives a runtime error (which is difficult to test)
+def can_find_global_arg():
+    """
+    >>> can_find_global_arg()
+    True
+    """
+    return __global_arg
+
+def cant_find_global_arg():
+    """
+    Gets _D_global_arg instead
+    >>> cant_find_global_arg()
+    False
+    """
+    class D:
+        def f(self):
+            return __global_arg
+    return D().f()
+
+class CMultiplyNested:
+    def f1(self, __arg, name=None, return_closure=False):
+        """
+        >>> inst = CMultiplyNested()
+        >>> for name in [None, '__arg', '_CMultiplyNested__arg', '_D__arg']:
+        ...    try:
+        ...        print(inst.f1(1,name))
+        ...    except TypeError:
+        ...        print("TypeError") # not concerned about exact details
+        ...    # now test behaviour is the same in closures
+        ...    closure = inst.f1(1, return_closure=True)
+        ...    try:
+        ...        if name is None:
+        ...            print(closure(2))
+        ...        else:
+        ...            print(closure(**{ name: 2}))
+        ...    except TypeError:
+        ...        print("TypeError")
+        2
+        2
+        TypeError
+        TypeError
+        TypeError
+        TypeError
+        2
+        2
+        """
+        class D:
+            def g(self, __arg):
+                return __arg
+        if return_closure:
+            return D().g
+        if name is not None:
+            return D().g(**{ name: 2 })
+        else:
+            return D().g(2)
+
+    def f2(self, __arg1):
+        """
+        This finds the global name '_D__arg1'
+        It's tested in this way because without the global
+        Python gives a runtime error and Cython a compile error
+        >>> print(CMultiplyNested().f2(1))
+        None
+        """
+        class D:
+            def g(self):
+                return __arg1
+        return D().g()
+
+    def f3(self, arg, name):
+        """
+        >>> inst = CMultiplyNested()
+        >>> inst.f3(1, None)
+        2
+        >>> inst.f3(1, '__arg') # doctest: +IGNORE_EXCEPTION_DETAIL
+        Traceback (most recent call last):
+        ...
+        TypeError:
+        >>> inst.f3(1, '_CMultiplyNested__arg')
+        2
+        """
+        def g(__arg, dummy=1):
+            return __arg
+        if name is not None:
+            return g(**{ name: 2})
+        else:
+            return g(2)
+
+    def f4(self, __arg):
+        """
+        >>> CMultiplyNested().f4(1)
+        1
+        """
+        def g():
+            return __arg
+        return g()
+
+    def f5(self, __arg):
+        """
+        Default values are found in the outer scope correcly
+        >>> CMultiplyNested().f5(1)
+        1
+        """
+        def g(x=__arg):
+            return x
+        return g()
+
+    def f6(self, __arg1):
+        """
+        This will find the global name _D__arg1
+        >>> print(CMultiplyNested().f6(1))
+        None
+        """
+        class D:
+            def g(self, x=__arg1):
+                return x
+        return D().g()
+
+    def f7(self, __arg):
+        """
+        Lookup works in generator expressions
+        >>> list(CMultiplyNested().f7(1))
+        [1]
+        """
+        return (__arg for x in range(1))
+
+class __NameWithDunder:
+    """
+    >>> __NameWithDunder.__name__
+    '__NameWithDunder'
+    """
+    pass
+
+class Inherits(__NameWithDunder):
+    """
+    Compile check that it can find the base class
+    >>> x = Inherits()
+    """
+    pass
+
+def regular_function(__x, dummy=None):
+    # as before, dummy stops Cython creating a 1 arg, non-keyword call
+    return __x
+
+class CallsRegularFunction:
+    def call(self):
+        """
+        >>> CallsRegularFunction().call()
+        1
+        """
+        return regular_function(__x=1)  # __x shouldn't be mangled as an argument elsewhere
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_unknown_names.py cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_unknown_names.py
--- cython-0.20.1+1~201611251650-6686/tests/run/methodmangling_unknown_names.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/methodmangling_unknown_names.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: allow_unknown_names, pure2.0, pure3.0
+
+class Test(object):
+    def run(self):
+        """
+        >>> Test().run()
+        NameError1
+        NameError2
+        found mangled
+        """
+        try:
+            print(__something)
+        except NameError:
+            print("NameError1")  # correct - shouldn't exist
+        globals()['__something'] = 'found unmangled'
+        try:
+            print(__something)
+        except NameError:
+            print("NameError2")  # correct - shouldn't exist
+        globals()['_Test__something'] = 'found mangled'
+        try:
+            print(__something)  # should print this
+        except NameError:
+            print("NameError3")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/method_module_name_T422.pyx cython-0.20.1+1~202203241016-9537/tests/run/method_module_name_T422.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/method_module_name_T422.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/method_module_name_T422.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 422
+# ticket: t422
 
 """
 >>> Foo.incr.__module__ is not None
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/modop.pyx cython-0.20.1+1~202203241016-9537/tests/run/modop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/modop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/modop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -7,11 +7,36 @@
     1
     >>> modobj('%d', 5)
     '5'
+    >>> modobj(1, 0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: integer... modulo by zero
     """
     obj1 = obj2 % obj3
     return obj1
 
 
+def mod_10_obj(int2):
+    """
+    >>> mod_10_obj(0)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ZeroDivisionError: ... modulo by zero
+    >>> 10 % 1
+    0
+    >>> mod_10_obj(1)
+    0
+    >>> mod_10_obj(3)
+    1
+    >>> 10 % -1
+    0
+    >>> mod_10_obj(-1)
+    0
+    >>> mod_10_obj(-10)
+    0
+    """
+    int1 = 10 % int2
+    return int1
+
+
 def mod_obj_10(int2):
     """
     >>> 0 % 10
@@ -154,6 +179,53 @@
     return int1
 
 
+def mod_int_17(int int2):
+    """
+    >>> 0 % 17
+    0
+    >>> mod_int_17(0)
+    0
+    >>> 1 % 17
+    1
+    >>> mod_int_17(1)
+    1
+    >>> (-1) % 17
+    16
+    >>> mod_int_17(-1)
+    16
+    >>> 9 % 17
+    9
+    >>> mod_int_17(16)
+    16
+    >>> 17 % 17
+    0
+    >>> mod_int_17(17)
+    0
+    >>> (-17) % 17
+    0
+    >>> mod_int_17(-17)
+    0
+    >>> (-18) % 17
+    16
+    >>> mod_int_17(-18)
+    16
+    >>> 10002 % 17
+    6
+    >>> mod_int_17(10002)
+    6
+    >>> int((2**25) % 17)
+    2
+    >>> int(mod_int_17(2**25))
+    2
+    >>> int((-2**25) % 17)
+    15
+    >>> int(mod_int_17(-2**25))
+    15
+    """
+    int1 = int2 % 17
+    return int1
+
+
 def mod_obj_m2(int2):
     """
     >>> 0 % -2
@@ -212,3 +284,12 @@
     str3 = "eggs"
     obj1 = str2 % str3  # '%' operator doesn't work on byte strings in Py3
     return obj1
+
+
+def mod_bigint(obj):
+    """
+    >>> print(mod_bigint(3316000000000))
+    319
+    """
+    result = obj % 999
+    return result
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/mod__spec__.pyx cython-0.20.1+1~202203241016-9537/tests/run/mod__spec__.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/mod__spec__.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/mod__spec__.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,51 @@
+# mode: run
+# tag: pep489
+
+import os.path
+
+module_spec = __spec__
+module_file = __file__
+
+
+def check_spec(spec=__spec__):
+    """
+    >>> check_spec()
+    """
+    assert __spec__ is not None
+    assert __spec__ is spec
+
+    assert __name__
+    assert __name__ == spec.name
+
+    assert spec.loader is not None
+    assert spec.loader is __loader__
+
+    assert not spec.parent
+    assert not __package__
+
+    assert spec.origin
+    assert spec.origin == module_file
+    assert spec.origin == __file__
+    assert os.path.basename(spec.origin).startswith(__name__)
+
+
+# validate that ModuleSpec is already complete at module initialisation time
+check_spec()
+check_spec(__spec__)
+check_spec(module_spec)
+
+
+def file_in_module():
+    """
+    >>> print(file_in_module())
+    mod__spec__
+    """
+    return os.path.basename(module_file).split('.', 1)[0]
+
+
+def file_in_function():
+    """
+    >>> print(file_in_function())
+    mod__spec__
+    """
+    return os.path.basename(__file__).split('.', 1)[0]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/mulop.pyx cython-0.20.1+1~202203241016-9537/tests/run/mulop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/mulop.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/mulop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,166 @@
+# mode: run
+# tag: multiply
+
+import sys
+IS_PY2 = sys.version_info[0] < 3
+
+
+def print_long(x):
+    if IS_PY2:
+        x = str(x).rstrip('L')
+    print(x)
+
+
+def mul_10_obj(x):
+    """
+    >>> mul_10_obj(0)
+    0
+    >>> mul_10_obj(10)
+    100
+    >>> mul_10_obj(-10)
+    -100
+    >>> 10 * (2**14)
+    163840
+    >>> mul_10_obj(2**14)
+    163840
+    >>> mul_10_obj(-2**14)
+    -163840
+    >>> print_long(10 * (2**29))
+    5368709120
+    >>> print_long(mul_10_obj(2**29))
+    5368709120
+    >>> print_long(mul_10_obj(-2**29))
+    -5368709120
+    >>> print_long(10 * (2**30))
+    10737418240
+    >>> print_long(mul_10_obj(2**30))
+    10737418240
+    >>> print_long(mul_10_obj(-2**30))
+    -10737418240
+    >>> print_long(10 * (2**63))
+    92233720368547758080
+    >>> print_long(mul_10_obj(2**63))
+    92233720368547758080
+    >>> print_long(mul_10_obj(-2**63))
+    -92233720368547758080
+    >>> print_long(10 * (2**128))
+    3402823669209384634633746074317682114560
+    >>> print_long(mul_10_obj(2**128))
+    3402823669209384634633746074317682114560
+    >>> print_long(mul_10_obj(-2**128))
+    -3402823669209384634633746074317682114560
+    """
+    result = 10 * x
+    return result
+
+
+def mul_obj_10(x):
+    """
+    >>> mul_obj_10(0)
+    0
+    >>> mul_obj_10(10)
+    100
+    >>> mul_obj_10(-10)
+    -100
+    >>> 10 * (2**14)
+    163840
+    >>> mul_obj_10(2**14)
+    163840
+    >>> mul_obj_10(-2**14)
+    -163840
+    >>> print_long(10 * (2**29))
+    5368709120
+    >>> print_long(mul_obj_10(2**29))
+    5368709120
+    >>> print_long(mul_obj_10(-2**29))
+    -5368709120
+    >>> print_long(10 * (2**30))
+    10737418240
+    >>> print_long(mul_obj_10(2**30))
+    10737418240
+    >>> print_long(mul_obj_10(-2**30))
+    -10737418240
+    >>> print_long(10 * (2**63))
+    92233720368547758080
+    >>> print_long(mul_obj_10(2**63))
+    92233720368547758080
+    >>> print_long(mul_obj_10(-2**63))
+    -92233720368547758080
+    >>> print_long(10 * (2**128))
+    3402823669209384634633746074317682114560
+    >>> print_long(mul_obj_10(2**128))
+    3402823669209384634633746074317682114560
+    >>> print_long(mul_obj_10(-2**128))
+    -3402823669209384634633746074317682114560
+    """
+    result = x * 10
+    return result
+
+
+def mul_bigint_obj(x):
+    """
+    >>> mul_bigint_obj(0)
+    0
+    >>> print_long(mul_bigint_obj(1))
+    536870912
+    >>> print_long(mul_bigint_obj(2))
+    1073741824
+    >>> print_long(mul_bigint_obj(2**29))
+    288230376151711744
+    >>> print_long(mul_bigint_obj(-2**29))
+    -288230376151711744
+    >>> print_long(mul_bigint_obj(2**30))
+    576460752303423488
+    >>> print_long(mul_bigint_obj(-2**30))
+    -576460752303423488
+    >>> print_long(mul_bigint_obj(2**59))
+    309485009821345068724781056
+    >>> print_long(mul_bigint_obj(-2**59))
+    -309485009821345068724781056
+    """
+    result = (2**29) * x
+    return result
+
+
+def mul_obj_float(x):
+    """
+    >>> mul_obj_float(-0.0)
+    -0.0
+    >>> mul_obj_float(0)
+    0.0
+    >>> mul_obj_float(1.0)
+    2.0
+    >>> mul_obj_float(-2.0)
+    -4.0
+    >>> mul_obj_float(-0.5)
+    -1.0
+    """
+    result = x * 2.0
+    return result
+
+
+def mul_float_obj(x):
+    """
+    >>> mul_float_obj(0)
+    0.0
+    >>> mul_float_obj(2)
+    4.0
+    >>> mul_float_obj(-2)
+    -4.0
+    >>> 2.0 * (2**30-1)
+    2147483646.0
+    >>> mul_float_obj(2**30-1)
+    2147483646.0
+    >>> mul_float_obj(-(2**30-1))
+    -2147483646.0
+    >>> mul_float_obj(-0.0)
+    -0.0
+    >>> mul_float_obj(1.0)
+    2.0
+    >>> mul_float_obj(-2.0)
+    -4.0
+    >>> mul_float_obj(-0.5)
+    -1.0
+    """
+    result = 2.0 * x
+    return result
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/no_gc_clear.pyx cython-0.20.1+1~202203241016-9537/tests/run/no_gc_clear.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/no_gc_clear.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/no_gc_clear.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,7 +3,7 @@
 tp_clear slot so that __dealloc__ will still see the original reference
 contents.
 
-Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14986
+Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14986
 """
 
 cimport cython
@@ -56,6 +56,21 @@
             pto.tp_clear(self)
 
 
+cdef class ReallowTpClear(DisableTpClear):
+    """
+    >>> import gc
+    >>> obj = ReallowTpClear()
+    >>> is_tp_clear_null(obj)
+    False
+
+    >>> obj.attr = obj  # create hard reference cycle
+    >>> del obj; _ignore = gc.collect()
+
+    # Problem: cannot really validate that the cycle was cleaned up without using weakrefs etc...
+    """
+    cdef public object attr
+
+
 def test_closure_without_clear(str x):
     """
     >>> c = test_closure_without_clear('abc')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/nogil_conditional.pyx cython-0.20.1+1~202203241016-9537/tests/run/nogil_conditional.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/nogil_conditional.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/nogil_conditional.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,271 @@
+# mode: run
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+
+def test(int x):
+    """
+    >>> test(0)
+    110
+    """
+    with nogil(True):
+        x = f_nogil(x)
+        with gil(True):
+            x = f_gil(x)
+    return x
+
+
+cdef int f_nogil(int x) nogil:
+    cdef int y
+    y = x + 10
+    return y
+
+
+def f_gil(x):
+    y = 0
+    y = x + 100
+    return y
+
+
+cdef int with_gil_func() except? -1 with gil:
+    raise Exception("error!")
+
+
+cdef int nogil_func() nogil except? -1:
+    with_gil_func()
+
+
+def test_nogil_exception_propagation():
+    """
+    >>> test_nogil_exception_propagation()
+    Traceback (most recent call last):
+       ...
+    Exception: error!
+    """
+    with nogil:
+        with gil:
+            with nogil(True):
+                nogil_func()
+
+
+cdef int write_unraisable() nogil:
+    with gil:
+        raise ValueError()
+
+
+def test_unraisable():
+    """
+    >>> print(test_unraisable())  # doctest: +ELLIPSIS
+    ValueError
+    Exception...ignored...
+    """
+    import sys
+    old_stderr = sys.stderr
+    stderr = sys.stderr = StringIO()
+    try:
+        write_unraisable()
+    finally:
+        sys.stderr = old_stderr
+    return stderr.getvalue().strip()
+
+
+def test_nested():
+    """
+    >>> test_nested()
+    240
+    """
+    cdef int res = 0
+
+    with nogil(True):
+        res = f_nogil(res)
+        with gil(1 < 2):
+            res = f_gil(res)
+            with nogil:
+                res = f_nogil(res)
+
+        with gil:
+            res = f_gil(res)
+            with nogil(True):
+                res = f_nogil(res)
+            with nogil:
+                res = f_nogil(res)
+
+    return res
+
+
+DEF FREE_GIL = True
+DEF FREE_GIL_FALSE = False
+
+
+def test_nested_condition_false():
+    """
+    >>> test_nested_condition_false()
+    220
+    """
+    cdef int res = 0
+
+    with gil(FREE_GIL_FALSE):
+        res = f_gil(res)
+        with nogil(False):
+            res = f_gil(res)
+
+        with nogil(FREE_GIL):
+            res = f_nogil(res)
+            with gil(False):
+                res = f_nogil(res)
+
+    return res
+
+def test_try_finally():
+    """
+    >>> test_try_finally()
+    113
+    """
+    cdef int res = 0
+
+    try:
+        with nogil(True):
+            try:
+                res = f_nogil(res)
+                with gil(1 < 2):
+                    try:
+                        res = f_gil(res)
+                    finally:
+                        res += 1
+            finally:
+                res = res + 1
+    finally:
+        res += 1
+
+    return res
+
+
+ctypedef fused number_or_object:
+    int
+    float
+    object
+
+
+def test_fused(number_or_object x) -> number_or_object:
+    """
+    >>> test_fused[int](1)
+    2
+    >>> test_fused[float](1.0)
+    2.0
+    >>> test_fused[object](1)
+    2
+    >>> test_fused[object](1.0)
+    2.0
+    """
+    cdef number_or_object res = x
+
+    with nogil(number_or_object is not object):
+        res = res + 1
+
+    return res
+
+
+ctypedef fused int_or_object:
+    int
+    object
+
+
+def test_fused_object(int_or_object x):
+    """
+    >>> test_fused_object[object]("spam")
+    456
+    >>> test_fused_object[int](1000)
+    1000
+    """
+    cdef int res = 0
+
+    if int_or_object is object:
+        with nogil(False):
+            res += len(x)
+
+        try:
+            with nogil(int_or_object is object):
+                try:
+                    with gil(int_or_object is object):
+                        res = f_gil(res)
+                    with gil:
+                        res = f_gil(res)
+                    with gil(False):
+                        res = f_nogil(res)
+
+                    with gil(int_or_object is not object):
+                        res = f_nogil(res)
+                    with nogil(False):
+                        res = f_nogil(res)
+
+                    res = f_nogil(res)
+                finally:
+                    res = res + 1
+
+            with nogil(int_or_object is not object):
+                res = f_gil(res)
+
+            with gil(int_or_object is not object):
+                res = f_gil(res)
+
+                with nogil(int_or_object is object):
+                    res = f_nogil(res)
+
+        finally:
+            res += 1
+    else:
+        res = x
+
+    return res
+
+
+def test_fused_int(int_or_object x):
+    """
+    >>> test_fused_int[object]("spam")
+    4
+    >>> test_fused_int[int](1000)
+    1452
+    """
+    cdef int res = 0
+
+    if int_or_object is int:
+        res += x
+
+        try:
+            with nogil(int_or_object is int):
+                try:
+                    with gil(int_or_object is int):
+                        res = f_gil(res)
+                    with gil:
+                        res = f_gil(res)
+                    with gil(False):
+                        res = f_nogil(res)
+
+                    with gil(int_or_object is not int):
+                        res = f_nogil(res)
+                    with nogil(False):
+                        res = f_nogil(res)
+
+                    res = f_nogil(res)
+                finally:
+                    res = res + 1
+
+            with nogil(int_or_object is not int):
+                res = f_gil(res)
+
+            with gil(int_or_object is not int):
+                res = f_gil(res)
+
+                with nogil(int_or_object is int):
+                    res = f_nogil(res)
+
+        finally:
+            res += 1
+    else:
+        with nogil(False):
+            res = len(x)
+
+    return res
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/nogil.pyx cython-0.20.1+1~202203241016-9537/tests/run/nogil.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/nogil.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/nogil.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -28,10 +28,50 @@
         y = x + 42
         return y
 
-cdef int with_gil_func() except 0 with gil:
+cdef void release_gil_in_nogil() nogil:
+    # This should generate valid code with/without the GIL
+    with nogil:
+        pass
+
+cpdef void release_gil_in_nogil2() nogil:
+    # This should generate valid code with/without the GIL
+    with nogil:
+        pass
+
+def test_release_gil_in_nogil():
+    """
+    >>> test_release_gil_in_nogil()
+    """
+    with nogil:
+        release_gil_in_nogil()
+    with nogil:
+        release_gil_in_nogil2()
+    release_gil_in_nogil()
+    release_gil_in_nogil2()
+
+cdef void get_gil_in_nogil() nogil:
+    with gil:
+        pass
+
+cpdef void get_gil_in_nogil2() nogil:
+    with gil:
+        pass
+
+def test_get_gil_in_nogil():
+    """
+    >>> test_get_gil_in_nogil()
+    """
+    with nogil:
+        get_gil_in_nogil()
+    with nogil:
+        get_gil_in_nogil2()
+    get_gil_in_nogil()
+    get_gil_in_nogil2()
+
+cdef int with_gil_func() except -1 with gil:
     raise Exception("error!")
 
-cdef int nogil_func() nogil except 0:
+cdef int nogil_func() nogil except -1:
     with_gil_func()
 
 def test_nogil_exception_propagation():
@@ -53,6 +93,7 @@
 def test_unraisable():
     """
     >>> print(test_unraisable())  # doctest: +ELLIPSIS
+    ValueError
     Exception...ignored...
     """
     import sys
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/non_dict_kwargs_T470.pyx cython-0.20.1+1~202203241016-9537/tests/run/non_dict_kwargs_T470.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/non_dict_kwargs_T470.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/non_dict_kwargs_T470.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 470
+# ticket: t470
 
 
 def func(**kwargs):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/non_future_division.pyx cython-0.20.1+1~202203241016-9537/tests/run/non_future_division.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/non_future_division.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/non_future_division.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -143,3 +143,73 @@
     (0, 2)
     """
     return a/b, b/a
+
+
+def div_by_0(a):
+    """
+    >>> div_by_0(0)
+    'OK'
+    >>> div_by_0(0.0)
+    'OK'
+    """
+    try:
+        1/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 1"
+    try:
+        1//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 2"
+    try:
+        5.0/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 3"
+    try:
+        5.0//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 4"
+    try:
+        5/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 5"
+    try:
+        5//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 6"
+    try:
+        (2**15)/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 7"
+    try:
+        (2**15)//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 8"
+    try:
+        (2**30)/a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 9"
+    try:
+        (2**30)//a
+    except ZeroDivisionError:
+        pass
+    else:
+        return "FAIL 10"
+    return 'OK'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_attributes.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_attributes.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_attributes.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_attributes.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,87 @@
+# mode: run
+# tag: numpy
+
+cimport cython
+
+import numpy as np
+cimport numpy as cnp
+
+cnp.import_array()
+
+
+@cython.test_assert_path_exists(
+    "//ReturnStatNode",
+    "//ReturnStatNode//IndexNode",
+    "//ReturnStatNode//IndexNode//SimpleCallNode",
+)
+@cython.test_fail_if_path_exists(
+    "//ReturnStatNode//AttributeNode",
+)
+def access_shape():
+    """
+    >>> print(access_shape())
+    10
+    """
+    cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \
+        1e10 * np.ones((10, 10))
+
+    return array_in.shape[0]
+
+
+@cython.test_assert_path_exists(
+    "//ReturnStatNode",
+    "//ReturnStatNode//SimpleCallNode",
+)
+@cython.test_fail_if_path_exists(
+    "//ReturnStatNode//AttributeNode",
+)
+def access_size():
+    """
+    >>> print(access_size())
+    100
+    """
+    cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \
+        1e10 * np.ones((10, 10))
+
+    return array_in.size
+
+
+@cython.test_assert_path_exists(
+    "//ReturnStatNode",
+    "//ReturnStatNode//IndexNode",
+    "//ReturnStatNode//IndexNode//SimpleCallNode",
+)
+@cython.test_fail_if_path_exists(
+    "//ReturnStatNode//AttributeNode",
+)
+def access_strides():
+    """
+    >>> x, y = access_strides()
+    >>> print(x)
+    80
+    >>> print(y)
+    8
+    """
+    cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \
+        1e10 * np.ones((10, 10), dtype=np.float64)
+
+    return (array_in.strides[0], array_in.strides[1])
+
+
+@cython.test_assert_path_exists(
+    "//ReturnStatNode",
+    "//ReturnStatNode//PrimaryCmpNode",
+    "//ReturnStatNode//PrimaryCmpNode//SimpleCallNode",
+)
+@cython.test_fail_if_path_exists(
+    "//ReturnStatNode//AttributeNode",
+)
+def access_data():
+    """
+    >>> access_data()
+    True
+    """
+    cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \
+        1e10 * np.ones((10, 10), dtype=np.float64)
+
+    return array_in.data is not NULL
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_bufacc_T155.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_bufacc_T155.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_bufacc_T155.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_bufacc_T155.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 155
+# ticket: t155
 # tag: numpy
 
 """
@@ -17,4 +17,3 @@
         A[i, :] /= 2
     return A[0,0]
 
-include "numpy_common.pxi"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_1.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_1.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_1.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_1.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: warnings, numpy
+
+cimport numpy as np
+# np.import_array not called - should generate warning
+
+cdef extern from *:
+    """
+    static void** _check_array_api(void) {
+        return PyArray_API; /* should be non NULL */
+    }
+    """
+    void** _check_array_api()
+
+def check_array_api():
+    """
+    >>> check_array_api()
+    True
+    """
+    return _check_array_api() != NULL
+
+
+_WARNINGS = """
+4:8: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called.
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_2.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_2.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_2.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_2.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: warnings, numpy
+
+cimport numpy as np
+np.import_array()
+# np.import_array is called - no warning necessary
+
+cdef extern from *:
+    """
+    static void** _check_array_api(void) {
+        return PyArray_API; /* should be non NULL */
+    }
+    """
+    void** _check_array_api()
+
+def check_array_api():
+    """
+    >>> check_array_api()
+    True
+    """
+    return _check_array_api() != NULL
+
+
+_WARNINGS = """
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_3.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_3.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_3.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_3.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,8 @@
+# mode: compile
+# tag: warnings, numpy
+
+import numpy as np
+# Numpy is only imported - no warning necessary
+
+_WARNINGS = """
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_4.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_4.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_4.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_4.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,24 @@
+# mode: run
+# tag: warnings, numpy
+
+cimport numpy
+numpy.import_array # dummy call should stop Cython auto-generating call to import_array
+
+cdef extern from *:
+    """
+    static void** _check_array_api(void) {
+        return PyArray_API; /* should be non NULL if initialized */
+    }
+    """
+    void** _check_array_api()
+
+def check_array_api():
+    """
+    >>> check_array_api()
+    True
+    """
+    return _check_array_api() == NULL # not initialized
+
+
+_WARNINGS = """
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_5.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_5.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_5.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_5.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: warnings, numpy
+
+from numpy cimport ndarray
+# np.import_array not called - should generate warning
+
+cdef extern from *:
+    """
+    static void** _check_array_api(void) {
+        return PyArray_API; /* should be non NULL */
+    }
+    """
+    void** _check_array_api()
+
+def check_array_api():
+    """
+    >>> check_array_api()
+    True
+    """
+    return _check_array_api() != NULL
+
+
+_WARNINGS = """
+4:0: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called.
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_6.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_6.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport_6.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport_6.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: warnings, numpy
+
+from numpy cimport ndarray, import_array
+import_array()
+# np.import_array is called - no warning necessary
+
+cdef extern from *:
+    """
+    static void** _check_array_api(void) {
+        return PyArray_API; /* should be non NULL */
+    }
+    """
+    void** _check_array_api()
+
+def check_array_api():
+    """
+    >>> check_array_api()
+    True
+    """
+    return _check_array_api() != NULL
+
+
+_WARNINGS = """
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_cimport.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_cimport.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -6,4 +6,3 @@
 True
 """
 cimport numpy as np
-include "numpy_common.pxi"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_common.pxi cython-0.20.1+1~202203241016-9537/tests/run/numpy_common.pxi
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_common.pxi	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_common.pxi	1970-01-01 00:00:00.000000000 +0000
@@ -1,10 +0,0 @@
-# hack to avoid C compiler warnings about unused functions in the NumPy header files
-
-cdef extern from *:
-   bint FALSE "0"
-   void import_array()
-#   void import_umath()
-
-if FALSE:
-    import_array()
-#    import_umath()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_import_array_error.srctree cython-0.20.1+1~202203241016-9537/tests/run/numpy_import_array_error.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_import_array_error.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_import_array_error.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,40 @@
+PYTHON setup.py build_ext -i
+PYTHON main.py
+
+############# setup.py ############
+
+from distutils.core import setup
+from Cython.Build import cythonize
+
+setup(ext_modules = cythonize('cimport_numpy.pyx'))
+
+############# numpy.pxd ############
+
+# A fake Numpy module. This defines a version of _import_array
+# that always fails. The Cython-generated call to _import_array
+# happens quite early (before the stringtab is initialized)
+# and thus the error itself handling could cause a segmentation fault
+# https://github.com/cython/cython/issues/4377
+
+cdef extern from *:
+    """
+    #define NPY_FEATURE_VERSION
+    static int _import_array(void) {
+        PyErr_SetString(PyExc_ValueError, "Oh no!");
+        return -1;
+    }
+    """
+    int _import_array() except -1
+    
+############# cimport_numpy.pyx ###########
+
+cimport numpy
+
+############# main.py ####################
+
+try:
+    import cimport_numpy
+except ImportError as e:
+    print(e)
+else:
+    assert(False)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_math.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_math.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_math.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_math.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -37,8 +37,8 @@
     assert not npmath.isnan(d_zero)
     assert not npmath.isnan(f_zero)
 
-    assert npmath.isinf(npmath.INFINITY) == 1
-    assert npmath.isinf(-npmath.INFINITY) == -1
+    assert npmath.isinf(-npmath.INFINITY)
+    assert npmath.isinf(npmath.INFINITY)
     assert npmath.isnan(npmath.NAN)
 
     assert npmath.signbit(npmath.copysign(1., -1.))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_parallel.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_parallel.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_parallel.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_parallel.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -4,7 +4,6 @@
 cimport cython
 from cython.parallel import prange
 cimport numpy as np
-include "numpy_common.pxi"
 
 
 @cython.boundscheck(False)
@@ -22,7 +21,7 @@
     3
     4
     """
-    cdef Py_ssize_t i
+    cdef Py_ssize_t i, length
     cdef np.ndarray[np.int_t] x
 
     try:
@@ -33,10 +32,10 @@
         return
 
     x = numpy.zeros(10, dtype=numpy.int)
+    length = x.shape[0]
 
-    for i in prange(x.shape[0], nogil=True):
+    for i in prange(length, nogil=True):
         x[i] = i - 5
 
     for i in x:
-        print i
-
+        print(i)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_pythran.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_pythran.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_pythran.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_pythran.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,67 @@
+# mode: run
+# tag: pythran, numpy, cpp
+# cython: np_pythran=True
+
+import numpy as np
+cimport numpy as cnp
+
+def diffuse():
+    """
+    >>> u = diffuse()
+    >>> count_non_zero = np.sum(u > 0)
+    >>> 850 < count_non_zero < (2**5) * (2**5) or count_non_zero
+    True
+    """
+    lx, ly = (2**5, 2**5)
+    u = np.zeros([lx, ly], dtype=np.double)
+    u[lx // 2, ly // 2] = 1000.0
+    _diffuse_numpy(u, 50)
+    return u
+
+
+def _diffuse_numpy(cnp.ndarray[double, ndim=2] u, int N):
+    """
+    Apply Numpy matrix for the Forward-Euler Approximation
+    """
+    cdef cnp.ndarray[double, ndim=2] temp = np.zeros_like(u)
+    mu = 0.1
+
+    for n in range(N):
+        temp[1:-1, 1:-1] = u[1:-1, 1:-1] + mu * (
+            u[2:, 1:-1] - 2 * u[1:-1, 1:-1] + u[0:-2, 1:-1] +
+            u[1:-1, 2:] - 2 * u[1:-1, 1:-1] + u[1:-1, 0:-2])
+        u[:, :] = temp[:, :]
+        temp[:, :] = 0.0
+
+
+def calculate_tax(cnp.ndarray[double, ndim=1] d):
+    """
+    >>> mu, sigma = 10.64, .35
+    >>> np.random.seed(1234)
+    >>> d = np.random.lognormal(mu, sigma, 10000)
+    >>> avg = calculate_tax(d)
+    >>> 0.243 < avg < 0.244 or avg  # 0.24342652180085891
+    True
+    """
+    tax_seg1 = d[(d > 256303)] * 0.45 - 16164.53
+    tax_seg2 = d[(d > 54057) & (d <= 256303)] * 0.42 - 8475.44
+    seg3 = d[(d > 13769) & (d <= 54057)] - 13769
+    seg4 = d[(d > 8820) & (d <= 13769)] - 8820
+    prog_seg3 = seg3 * 0.0000022376 + 0.2397
+    prog_seg4 = seg4 * 0.0000100727 + 0.14
+    return (
+        np.sum(tax_seg1) +
+        np.sum(tax_seg2) +
+        np.sum(seg3 * prog_seg3 + 939.57) +
+        np.sum(seg4 * prog_seg4)
+    ) / np.sum(d)
+
+def access_shape():
+    """
+    >>> access_shape()
+    10
+    """
+    cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \
+                    1e10 * np.ones((10, 10))
+
+    return array_in.shape[0]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_pythran_unit.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_pythran_unit.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_pythran_unit.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_pythran_unit.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,25 @@
+# mode: run
+# tag: pythran, numpy, cpp
+# cython: np_pythran=True
+
+import numpy as np
+cimport numpy as np
+
+def trigo(np.ndarray[double, ndim=1] angles):
+
+    """
+    >>> a = np.array([0., np.pi, np.pi *2])
+    >>> trigo(a)
+    array([ 1., -1.,  1.])
+    """
+    return np.cos(angles)
+
+def power(np.ndarray[double, ndim=1] values):
+
+    """
+    >>> a = np.array([0., 1., 2.])
+    >>> res = power(a)
+    >>> res[0], res[1], res[2]
+    (0.0, 1.0, 8.0)
+    """
+    return values ** 3
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_subarray.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_subarray.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_subarray.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_subarray.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,37 @@
+# tag: numpy
+
+cimport numpy as np
+cimport cython
+
+import numpy as py_numpy
+
+__doc__ = u"""
+
+    >>> test_record_subarray()
+    
+"""
+
+def test_record_subarray():
+    cdef np.ndarray x = py_numpy.zeros((2,2),
+                                       dtype=[('a', py_numpy.int32),
+                                              ('b', py_numpy.float64, (3, 3))])
+    cdef np.dtype descr   = x.dtype
+    cdef np.dtype a_descr = descr.fields['a'][0]
+    cdef np.dtype b_descr = descr.fields['b'][0]
+
+    # Make sure the dtype looks like we expect
+    assert descr.fields == {'a': (py_numpy.dtype('int32'), 0),
+                            'b': (py_numpy.dtype(('=f8', (3, 3))), 4)}, descr.fields
+
+    # Make sure that HASSUBARRAY is working
+    assert not np.PyDataType_HASSUBARRAY(descr)
+    assert not np.PyDataType_HASSUBARRAY(a_descr)
+    assert np.PyDataType_HASSUBARRAY(b_descr)
+
+    # Make sure the direct field access works
+    assert b_descr.subarray.shape == (3, 3), b_descr.subarray.shape
+
+    # Make sure the safe high-level helper function works
+    assert np.PyDataType_SHAPE(descr) == (), np.PyDataType_SHAPE(descr)
+    assert np.PyDataType_SHAPE(a_descr) == (), np.PyDataType_SHAPE(a_descr)
+    assert np.PyDataType_SHAPE(b_descr) == (3, 3), np.PyDataType_SHAPE(b_descr)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_test.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_test.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_test.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_test.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,28 +1,24 @@
+# mode: run
 # tag: numpy
-# cannot be named "numpy" in order to not clash with the numpy module!
 
 cimport numpy as np
 cimport cython
 
-import sys
+import numpy as np
+
+import re
 
-from libc.stdlib cimport malloc
 
 def little_endian():
     cdef int endian_detector = 1
     return (&endian_detector)[0] != 0
 
-__test__ = {}
 
 def testcase(f):
-    __test__[f.__name__] = f.__doc__
+    # testcase decorator now does nothing (following changes to doctest)
+    # but is a useful indicator of what functions are designed as tests
     return f
 
-def testcase_have_buffer_interface(f):
-    major, minor, *rest = np.__version__.split('.')
-    if (int(major), int(minor)) >= (1, 5) and sys.version_info[:2] >= (2, 6):
-        __test__[f.__name__] = f.__doc__
-    return f
 
 if little_endian():
     my_endian = '<'
@@ -31,32 +27,84 @@
     my_endian = '>'
     other_endian = '<'
 
-try:
-    import numpy as np
-    __doc__ = u"""
 
+def assert_dtype_sizes():
+    """
     >>> assert_dtype_sizes()
+    """
+    assert sizeof(np.int8_t) == 1
+    assert sizeof(np.int16_t) == 2
+    assert sizeof(np.int32_t) == 4
+    assert sizeof(np.int64_t) == 8
+    assert sizeof(np.uint8_t) == 1
+    assert sizeof(np.uint16_t) == 2
+    assert sizeof(np.uint32_t) == 4
+    assert sizeof(np.uint64_t) == 8
+    assert sizeof(np.float32_t) == 4
+    assert sizeof(np.float64_t) == 8
+    assert sizeof(np.complex64_t) == 8
+    assert sizeof(np.complex128_t) == 16
+
+
+@testcase
+def test_enums():
+    """
+    >>> test_enums()
+    """
+    cdef np.NPY_CASTING nc = np.NPY_NO_CASTING
+    assert nc != np.NPY_SAFE_CASTING
 
+
+def ndarray_str(arr):
+    u"""
+    Work around display differences in NumPy 1.14.
+    """
+    return re.sub(ur'\[ +', '[', unicode(arr))
+
+
+def basic():
+    """
     >>> basic()
     [[0 1 2 3 4]
      [5 6 7 8 9]]
     2 0 9 5
+    """
+    cdef object[int, ndim=2] buf = np.arange(10, dtype='i').reshape((2, 5))
+    print buf
+    print buf[0, 2], buf[0, 0], buf[1, 4], buf[1, 0]
 
-    >>> three_dim()
-    [[[  0.   1.   2.   3.]
-      [  4.   5.   6.   7.]]
-    <_BLANKLINE_>
-     [[  8.   9.  10.  11.]
-      [ 12.  13.  14.  15.]]
-    <_BLANKLINE_>
-     [[ 16.  17.  18.  19.]
-      [ 20.  21.  22.  23.]]]
+
+def three_dim():
+    """
+    >>> three_dim()  # doctest: +NORMALIZE_WHITESPACE
+    [[[0.   1.   2.   3.]
+      [4.   5.   6.   7.]]
+    
+     [[8.   9.  10.  11.]
+      [12.  13.  14.  15.]]
+    
+     [[16.  17.  18.  19.]
+      [20.  21.  22.  23.]]]
     6.0 0.0 13.0 8.0
+    """
+    cdef object[double, ndim=3] buf = np.arange(24, dtype='d').reshape((3,2,4))
+    print ndarray_str(buf)
+    print buf[0, 1, 2], buf[0, 0, 0], buf[1, 1, 1], buf[1, 0, 0]
+
 
+def obj_array():
+    """
     >>> obj_array()
     [a 1 {}]
     a 1 {}
+    """
+    cdef object[object, ndim=1] buf = np.array(["a", 1, {}])
+    print str(buf).replace('"', '').replace("'", '')
+    print buf[0], buf[1], buf[2]
+
 
+def print_long_2d(np.ndarray[long, ndim=2] arr):
+    """
     Test various forms of slicing, picking etc.
     >>> a = np.arange(10, dtype='l').reshape(2, 5)
     >>> print_long_2d(a)
@@ -86,9 +134,16 @@
     2 7
     3 8
     4 9
+    """
+    cdef int i, j
+    for i in range(arr.shape[0]):
+        print u" ".join([unicode(arr[i, j]) for j in range(arr.shape[1])])
 
+
+def put_range_long_1d(np.ndarray[long] arr):
+    """
     Write to slices
-    >>> b = a.copy()
+    >>> b = np.arange(10, dtype='l').reshape(2, 5)
     >>> put_range_long_1d(b[:, 3])
     >>> print (b)
     [[0 1 2 0 4]
@@ -109,7 +164,16 @@
     >>> print (a)
     [[0 0 0 0 0]
      [0 0 0 0 0]]
+    """
+    # Writes 0,1,2,... to array and returns array
+    cdef int value = 0, i
+    for i in range(arr.shape[0]):
+        arr[i] = value
+        value += 1
+
 
+def test_c_contig(np.ndarray[int, ndim=2, mode='c'] arr):
+    """
     Test contiguous access modes:
     >>> c_arr = np.array(np.arange(12, dtype='i').reshape(3,4), order='C')
     >>> f_arr = np.array(np.arange(12, dtype='i').reshape(3,4), order='F')
@@ -117,219 +181,41 @@
     0 1 2 3
     4 5 6 7
     8 9 10 11
-    >>> test_f_contig(f_arr)
-    0 1 2 3
-    4 5 6 7
-    8 9 10 11
     >>> test_c_contig(f_arr) #doctest: +ELLIPSIS
     Traceback (most recent call last):
        ...
     ValueError: ndarray is not C...contiguous
-    >>> test_f_contig(c_arr) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-       ...
-    ValueError: ndarray is not Fortran contiguous
     >>> test_c_contig(c_arr[::2,::2]) #doctest: +ELLIPSIS
     Traceback (most recent call last):
        ...
     ValueError: ndarray is not C...contiguous
-    
-    >>> test_dtype('b', inc1_byte)
-    >>> test_dtype('B', inc1_ubyte)
-    >>> test_dtype('h', inc1_short)
-    >>> test_dtype('H', inc1_ushort)
-    >>> test_dtype('i', inc1_int)
-    >>> test_dtype('I', inc1_uint)
-    >>> test_dtype('l', inc1_long)
-    >>> test_dtype('L', inc1_ulong)
-    
-    >>> test_dtype('f', inc1_float)
-    >>> test_dtype('d', inc1_double)
-    >>> test_dtype('g', inc1_longdouble)
-    >>> test_dtype('O', inc1_object)
-    >>> test_dtype('F', inc1_cfloat) # numpy format codes differ from buffer ones here
-    >>> test_dtype('D', inc1_cdouble)
-    >>> test_dtype('G', inc1_clongdouble)
-    >>> test_dtype('F', inc1_cfloat_struct)
-    >>> test_dtype('D', inc1_cdouble_struct)
-    >>> test_dtype('G', inc1_clongdouble_struct)
-
-    >>> test_dtype(np.int, inc1_int_t)
-    >>> test_dtype(np.longlong, inc1_longlong_t)
-    >>> test_dtype(np.float, inc1_float_t)
-    >>> test_dtype(np.double, inc1_double_t)
-    >>> test_dtype(np.intp, inc1_intp_t)
-    >>> test_dtype(np.uintp, inc1_uintp_t)
-
-    >>> test_dtype(np.longdouble, inc1_longdouble_t)
-
-    >>> test_dtype(np.int32, inc1_int32_t)
-    >>> test_dtype(np.float64, inc1_float64_t)
-
-    Endian tests:
-    >>> test_dtype('%si' % my_endian, inc1_int)
-    >>> test_dtype('%si' % other_endian, inc1_int)  #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-       ...
-    ValueError: ...
-    
-
-
-    >>> test_recordarray()
-    
-    >>> print(test_nested_dtypes(np.zeros((3,), dtype=np.dtype([\
-            ('a', np.dtype('i,i')),\
-            ('b', np.dtype('i,i'))\
-        ]))))
-    array([((0, 0), (0, 0)), ((1, 2), (1, 4)), ((1, 2), (1, 4))], 
-          dtype=[('a', [('f0', '!i4'), ('f1', '!i4')]), ('b', [('f0', '!i4'), ('f1', '!i4')])])
-
-    >>> print(test_nested_dtypes(np.zeros((3,), dtype=np.dtype([\
-            ('a', np.dtype('i,f')),\
-            ('b', np.dtype('i,i'))\
-        ]))))
-    Traceback (most recent call last):
-        ...
-    ValueError: Buffer dtype mismatch, expected 'int' but got 'float' in 'DoubleInt.y'
-
-    >>> print(test_packed_align(np.zeros((1,), dtype=np.dtype('b,i', align=False))))
-    [(22, 23)]
-
-
-    The output changed in Python 3:
-    >> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=True))))
-    array([(22, 23)], 
-          dtype=[('f0', '|i1'), ('', '|V3'), ('f1', '!i4')])
-
-    ->
-
-    array([(22, 23)], 
-          dtype={'names':['f0','f1'], 'formats':['i1','!i4'], 'offsets':[0,4], 'itemsize':8, 'aligned':True})
-
-
-    >>> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=True))))
-    [(22, 23)]
-
-    >>> print(test_packed_align(np.zeros((1,), dtype=np.dtype('b,i', align=True)))) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-        ...
-    ValueError: ...
-
-    >>> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=False)))) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-        ...
-    ValueError: ...
-
-
-    >>> test_good_cast()
-    True
-    >>> test_bad_cast()
-    Traceback (most recent call last):
-        ...
-    ValueError: Item size of buffer (1 byte) does not match size of 'int' (4 bytes)
-
-    >>> test_complextypes()
-    1,1
-    1,1
-    8,16
-
-    >>> test_point_record()
-    array([(0.0, 0.0), (1.0, -1.0), (2.0, -2.0)], 
-          dtype=[('x', '!f8'), ('y', '!f8')])
-
-"""
-
-    if np.__version__ >= '1.6' and False:
-        __doc__ += u"""
-        Tests are DISABLED as the buffer format parser does not align members
-        of aligned structs in padded structs in relation to the possibly
-        unaligned initial offset.
-
-        The following expose bugs in Numpy (versions prior to 2011-04-02):
-        >>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=True))))
-        array([(22, 23, (24, 25), 26)],
-              dtype=[('a', '|i1'), ('', '|V3'), ('b', '!i4'), ('sub', [('f0', '|i1'), ('f1', '!i4')]), ('', '|V3'), ('c', '!i4')])
-
-        >>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=True))]))))
-        array([(22, 23, 24, (27, 28))],
-              dtype=[('a', '|i1'), ('b', '!i4'), ('c', '|i1'), ('sub', [('f0', '|i1'), ('', '|V3'), ('f1', '!i4')])])
-
-        >>> print(test_partially_packed_align(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], align=False)))) #doctest: +ELLIPSIS
-        Traceback (most recent call last):
-            ...
-        ValueError: ...
-
-        >>> print(test_partially_packed_align_2(np.zeros((1,), dtype=np.dtype([('a', 'b'), ('b', 'i'), ('c', 'b'), ('sub', np.dtype('b,i', align=False))])))) #doctest: +ELLIPSIS
-        Traceback (most recent call last):
-            ...
-        ValueError: ...
-        """
-
-except:
-    __doc__ = u""
-
-__test__[__name__] = __doc__
-
-def assert_dtype_sizes():
-    assert sizeof(np.int8_t) == 1
-    assert sizeof(np.int16_t) == 2
-    assert sizeof(np.int32_t) == 4
-    assert sizeof(np.int64_t) == 8
-    assert sizeof(np.uint8_t) == 1
-    assert sizeof(np.uint16_t) == 2
-    assert sizeof(np.uint32_t) == 4
-    assert sizeof(np.uint64_t) == 8
-    assert sizeof(np.float32_t) == 4
-    assert sizeof(np.float64_t) == 8
-    assert sizeof(np.complex64_t) == 8
-    assert sizeof(np.complex128_t) == 16
-
-def ndarray_str(arr):
-    u"""
-    Since Py2.3 doctest don't support , manually replace blank lines
-    with <_BLANKLINE_>
     """
-    return unicode(arr).replace(u'\n\n', u'\n<_BLANKLINE_>\n')
-
-def basic():
-    cdef object[int, ndim=2] buf = np.arange(10, dtype='i').reshape((2, 5))
-    print buf
-    print buf[0, 2], buf[0, 0], buf[1, 4], buf[1, 0]
-
-def three_dim():
-    cdef object[double, ndim=3] buf = np.arange(24, dtype='d').reshape((3,2,4))
-    print ndarray_str(buf)
-    print buf[0, 1, 2], buf[0, 0, 0], buf[1, 1, 1], buf[1, 0, 0]
-
-def obj_array():
-    cdef object[object, ndim=1] buf = np.array(["a", 1, {}])
-    print str(buf).replace('"', '').replace("'", '')
-    print buf[0], buf[1], buf[2]
-
-
-def print_long_2d(np.ndarray[long, ndim=2] arr):
     cdef int i, j
     for i in range(arr.shape[0]):
         print u" ".join([unicode(arr[i, j]) for j in range(arr.shape[1])])
 
-def put_range_long_1d(np.ndarray[long] arr):
-    u"""Writes 0,1,2,... to array and returns array"""
-    cdef int value = 0, i
-    for i in range(arr.shape[0]):
-        arr[i] = value
-        value += 1
-
-def test_c_contig(np.ndarray[int, ndim=2, mode='c'] arr):
-    cdef int i, j
-    for i in range(arr.shape[0]):
-        print u" ".join([unicode(arr[i, j]) for j in range(arr.shape[1])])
 
 def test_f_contig(np.ndarray[int, ndim=2, mode='fortran'] arr):
+    """
+    Test contiguous access modes:
+    >>> c_arr = np.array(np.arange(12, dtype='i').reshape(3,4), order='C')
+    >>> f_arr = np.array(np.arange(12, dtype='i').reshape(3,4), order='F')
+    >>> test_f_contig(f_arr)
+    0 1 2 3
+    4 5 6 7
+    8 9 10 11
+    >>> test_f_contig(c_arr) #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+       ...
+    ValueError: ndarray is not Fortran contiguous
+    """
     cdef int i, j
     for i in range(arr.shape[0]):
         print u" ".join([unicode(arr[i, j]) for j in range(arr.shape[1])])
 
+
 # Exhaustive dtype tests -- increments element [1] by 1 (or 1+1j) for all dtypes
+def inc1_bool(np.ndarray[unsigned char] arr):           arr[1] += 1
 def inc1_byte(np.ndarray[char] arr):                    arr[1] += 1
 def inc1_ubyte(np.ndarray[unsigned char] arr):          arr[1] += 1
 def inc1_short(np.ndarray[short] arr):                  arr[1] += 1
@@ -352,7 +238,7 @@
 def inc1_cfloat_struct(np.ndarray[np.cfloat_t] arr):
     arr[1].real += 1
     arr[1].imag += 1
-    
+
 def inc1_cdouble_struct(np.ndarray[np.cdouble_t] arr):
     arr[1].real += 1
     arr[1].imag += 1
@@ -368,7 +254,6 @@
     o += 1
     arr[1] = o # unfortunately, += segfaults for objects
 
-
 def inc1_int_t(np.ndarray[np.int_t] arr):               arr[1] += 1
 def inc1_long_t(np.ndarray[np.long_t] arr):             arr[1] += 1
 def inc1_longlong_t(np.ndarray[np.longlong_t] arr):     arr[1] += 1
@@ -382,8 +267,49 @@
 def inc1_int32_t(np.ndarray[np.int32_t] arr):           arr[1] += 1
 def inc1_float64_t(np.ndarray[np.float64_t] arr):       arr[1] += 1
 
-    
+
 def test_dtype(dtype, inc1):
+    """
+    >>> test_dtype('?', inc1_bool)
+    >>> test_dtype('b', inc1_byte)
+    >>> test_dtype('B', inc1_ubyte)
+    >>> test_dtype('h', inc1_short)
+    >>> test_dtype('H', inc1_ushort)
+    >>> test_dtype('i', inc1_int)
+    >>> test_dtype('I', inc1_uint)
+    >>> test_dtype('l', inc1_long)
+    >>> test_dtype('L', inc1_ulong)
+
+    >>> test_dtype('f', inc1_float)
+    >>> test_dtype('d', inc1_double)
+    >>> test_dtype('g', inc1_longdouble)
+    >>> test_dtype('O', inc1_object)
+    >>> test_dtype('F', inc1_cfloat) # numpy format codes differ from buffer ones here
+    >>> test_dtype('D', inc1_cdouble)
+    >>> test_dtype('G', inc1_clongdouble)
+    >>> test_dtype('F', inc1_cfloat_struct)
+    >>> test_dtype('D', inc1_cdouble_struct)
+    >>> test_dtype('G', inc1_clongdouble_struct)
+
+    >>> test_dtype(np.int, inc1_int_t)
+    >>> test_dtype(np.longlong, inc1_longlong_t)
+    >>> test_dtype(np.float, inc1_float_t)
+    >>> test_dtype(np.double, inc1_double_t)
+    >>> test_dtype(np.intp, inc1_intp_t)
+    >>> test_dtype(np.uintp, inc1_uintp_t)
+
+    >>> test_dtype(np.longdouble, inc1_longdouble_t)
+
+    >>> test_dtype(np.int32, inc1_int32_t)
+    >>> test_dtype(np.float64, inc1_float64_t)
+
+    Endian tests:
+    >>> test_dtype('%si' % my_endian, inc1_int)
+    >>> test_dtype('%si' % other_endian, inc1_int)  #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+       ...
+    ValueError: ...
+    """
     if dtype in ("g", np.longdouble,
                  "G", np.clongdouble):
         if sizeof(double) == sizeof(long double): # MSVC
@@ -392,15 +318,26 @@
         a = np.array([0, 10+10j], dtype=dtype)
         inc1(a)
         if a[1] != (11 + 11j): print u"failed!", a[1]
+    elif dtype == '?':
+        # bool ndarrays coerce all values to 0 or 1
+        a = np.array([0, 0], dtype=dtype)
+        inc1(a)
+        if a[1] != 1: print u"failed!"
+        inc1(a)
+        if a[1] != 1: print u"failed!"
     else:
         a = np.array([0, 10], dtype=dtype)
         inc1(a)
         if a[1] != 11: print u"failed!"
 
+
 cdef struct DoubleInt:
     int x, y
 
 def test_recordarray():
+    """
+    >>> test_recordarray()
+    """
     cdef object[DoubleInt] arr
     arr = np.array([(5,5), (4, 6)], dtype=np.dtype('i,i'))
     cdef DoubleInt rec
@@ -416,6 +353,7 @@
     if arr[1].x != 5: print u"failed"
     if arr[1].y != 10: print u"failed"
 
+
 cdef struct NestedStruct:
     DoubleInt a
     DoubleInt b
@@ -429,6 +367,22 @@
     BadDoubleInt b
 
 def test_nested_dtypes(obj):
+    """
+    >>> print(test_nested_dtypes(np.zeros((3,), dtype=np.dtype([\
+            ('a', np.dtype('i,i')),\
+            ('b', np.dtype('i,i'))\
+        ]))))                              # doctest: +NORMALIZE_WHITESPACE
+    array([((0, 0), (0, 0)), ((1, 2), (1, 4)), ((1, 2), (1, 4))],
+          dtype=[('a', [('f0', '!i4'), ('f1', '!i4')]), ('b', [('f0', '!i4'), ('f1', '!i4')])])
+
+    >>> print(test_nested_dtypes(np.zeros((3,), dtype=np.dtype([\
+            ('a', np.dtype('i,f')),\
+            ('b', np.dtype('i,i'))\
+        ]))))
+    Traceback (most recent call last):
+        ...
+    ValueError: Buffer dtype mismatch, expected 'int' but got 'float' in 'DoubleInt.y'
+    """
     cdef object[NestedStruct] arr = obj
     arr[1].a.x = 1
     arr[1].a.y = 2
@@ -437,19 +391,36 @@
     arr[2] = arr[1]
     return repr(arr).replace('<', '!').replace('>', '!')
 
+
 def test_bad_nested_dtypes():
+    """
+    >>> test_bad_nested_dtypes()
+    """
     cdef object[BadNestedStruct] arr
 
+
 def test_good_cast():
+    """
+    >>> test_good_cast()
+    True
+    """
     # Check that a signed int can round-trip through casted unsigned int access
     cdef np.ndarray[unsigned int, cast=True] arr = np.array([-100], dtype='i')
     cdef unsigned int data = arr[0]
     return -100 == data
 
+
 def test_bad_cast():
+    """
+    >>> test_bad_cast()
+    Traceback (most recent call last):
+        ...
+    ValueError: Item size of buffer (1 byte) does not match size of 'int' (4 bytes)
+    """
     # This should raise an exception
     cdef np.ndarray[int, cast=True] arr = np.array([1], dtype='b')
 
+
 cdef packed struct PackedStruct:
     char a
     int b
@@ -471,16 +442,45 @@
     UnpackedStruct sub
 
 def test_packed_align(np.ndarray[PackedStruct] arr):
+    """
+    >>> print(test_packed_align(np.zeros((1,), dtype=np.dtype('b,i', align=False))))
+    [(22, 23)]
+    >>> print(test_packed_align(np.zeros((1,), dtype=np.dtype('b,i', align=True)))) #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    ValueError: ...
+    """
     arr[0].a = 22
     arr[0].b = 23
     return list(arr)
 
+
 def test_unpacked_align(np.ndarray[UnpackedStruct] arr):
+    """
+    The output changed in Python 3:
+    >> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=True))))
+    array([(22, 23)],
+          dtype=[('f0', '|i1'), ('', '|V3'), ('f1', '!i4')])
+
+    ->
+
+    array([(22, 23)],
+          dtype={'names':['f0','f1'], 'formats':['i1','!i4'], 'offsets':[0,4], 'itemsize':8, 'aligned':True})
+
+
+    >>> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=True))))
+    [(22, 23)]
+    >>> print(test_unpacked_align(np.zeros((1,), dtype=np.dtype('b,i', align=False)))) #doctest: +ELLIPSIS
+    Traceback (most recent call last):
+        ...
+    ValueError: ...
+    """
     arr[0].a = 22
     arr[0].b = 23
     # return repr(arr).replace('<', '!').replace('>', '!')
     return list(arr)
 
+
 def test_partially_packed_align(np.ndarray[PartiallyPackedStruct] arr):
     arr[0].a = 22
     arr[0].b = 23
@@ -489,6 +489,7 @@
     arr[0].c = 26
     return repr(arr).replace('<', '!').replace('>', '!')
 
+
 def test_partially_packed_align_2(np.ndarray[PartiallyPackedStruct2] arr):
     arr[0].a = 22
     arr[0].b = 23
@@ -497,7 +498,14 @@
     arr[0].sub.b = 28
     return repr(arr).replace('<', '!').replace('>', '!')
 
+
 def test_complextypes():
+    """
+    >>> test_complextypes()
+    1,1
+    1,1
+    8,16
+    """
     cdef np.complex64_t x64 = 1, y64 = 1j
     cdef np.complex128_t x128 = 1, y128 = 1j
     x64 = x64 + y64
@@ -511,6 +519,11 @@
     np.float64_t x, y
 
 def test_point_record():
+    """
+    >>> test_point_record()         # doctest: +NORMALIZE_WHITESPACE
+    array([(0., 0.), (1., -1.), (2., -2.)],
+          dtype=[('x', '!f8'), ('y', '!f8')])
+    """
     cdef np.ndarray[Point] test
     Point_dtype = np.dtype([('x', np.float64), ('y', np.float64)])
     test = np.zeros(3, Point_dtype)
@@ -518,7 +531,10 @@
     for i in range(3):
         test[i].x = i
         test[i].y = -i
-    print repr(test).replace('<', '!').replace('>', '!')
+    print re.sub(
+        r'\.0+\b', '.', repr(test).replace('<', '!').replace('>', '!')
+                                  .replace('( ', '(').replace(',  ', ', '))
+
 
 # Test fused np.ndarray dtypes and runtime dispatch
 @testcase
@@ -552,6 +568,7 @@
     np.float32_t
     np.float64_t
 
+
 @testcase
 def test_fused_external(np.ndarray[fused_external, ndim=1] a):
     """
@@ -570,6 +587,7 @@
     """
     print a.dtype
 
+
 cdef fused fused_buffers:
     np.ndarray[np.int32_t, ndim=1]
     np.int64_t[::1]
@@ -581,6 +599,7 @@
     ['int64_t[::1]', 'ndarray[int32_t,ndim=1]']
     """
 
+
 cpdef _fused_cpdef_buffers(np.ndarray[fused_external] a):
     print a.dtype
 
@@ -596,6 +615,7 @@
     cdef np.ndarray[np.int32_t] typed_array = int32_array
     _fused_cpdef_buffers(typed_array)
 
+
 @testcase
 def test_fused_ndarray_integral_dtype(np.ndarray[cython.integral, ndim=1] a):
     """
@@ -618,6 +638,7 @@
     # select different integer types with equal sizeof()
     print a[5], b[6]
 
+
 cdef fused fused_dtype:
     float complex
     double complex
@@ -654,11 +675,12 @@
     np.ndarray[Foo, ndim=1]
 
 def get_Foo_array():
-    cdef Foo[:] result =  malloc(sizeof(Foo) * 10)
-    result[5].b = 9.0
-    return np.asarray(result)
+    cdef Foo data[10]
+    for i in range(10):
+        data[i] = [0, 0]
+    data[5].b = 9.0
+    return np.asarray(data).copy()
 
-@testcase_have_buffer_interface
 def test_fused_ndarray(fused_ndarray a):
     """
     >>> import cython
@@ -683,6 +705,7 @@
     else:
         print b[5]
 
+
 cpdef test_fused_cpdef_ndarray(fused_ndarray a):
     """
     >>> import cython
@@ -707,9 +730,7 @@
     else:
         print b[5]
 
-testcase_have_buffer_interface(test_fused_cpdef_ndarray)
 
-@testcase_have_buffer_interface
 def test_fused_cpdef_ndarray_cdef_call():
     """
     >>> test_fused_cpdef_ndarray_cdef_call()
@@ -719,6 +740,7 @@
     cdef np.ndarray[Foo, ndim=1] foo_array = get_Foo_array()
     test_fused_cpdef_ndarray(foo_array)
 
+
 cdef fused int_type:
     np.int32_t
     np.int64_t
@@ -747,6 +769,7 @@
     """
     print a1[1], a2[2], a3[3], a4[4]
 
+
 ctypedef np.int32_t typedeffed_type
 
 cdef fused typedeffed_fused_type:
@@ -781,6 +804,7 @@
     """
     print a[3]
 
+
 # test fused memoryview slices
 cdef fused memslice_fused_dtype:
     float
@@ -811,6 +835,7 @@
     cdef memslice_fused_dtype[:] b = a
     print cython.typeof(a), cython.typeof(b), a[5], b[6]
 
+
 cdef fused memslice_fused:
     float[:]
     double[:]
@@ -840,6 +865,7 @@
     cdef memslice_fused b = a
     print cython.typeof(a), cython.typeof(b), a[5], b[6]
 
+
 @testcase
 def test_dispatch_memoryview_object():
     """
@@ -851,6 +877,7 @@
     cdef int[:] m3 =  m
     test_fused_memslice(m3)
 
+
 cdef fused ndim_t:
     double[:]
     double[:, :]
@@ -875,4 +902,65 @@
     """
     print cython.typeof(array), np.asarray(array).ndim
 
-include "numpy_common.pxi"
+
+@testcase
+def test_copy_buffer(np.ndarray[double, ndim=1] a):
+    """
+    >>> a = test_copy_buffer(np.ones(10, dtype=np.double))
+    >>> len(a)
+    10
+    >>> print(a.dtype)
+    float64
+    >>> a[0]
+    1.0
+    """
+    a = a.copy()
+    a = a.copy()
+    a = a.copy()
+    a = a.copy()
+    a = a.copy()
+    return a
+
+
+@testcase
+def test_broadcast_comparison(np.ndarray[double, ndim=1] a):
+    """
+    >>> a = np.ones(10, dtype=np.double)
+    >>> a0, obj0, a1, obj1 = test_broadcast_comparison(a)
+    >>> np.all(a0 == (a == 0)) or a0
+    True
+    >>> np.all(a1 == (a == 1)) or a1
+    True
+    >>> np.all(obj0 == (a == 0)) or obj0
+    True
+    >>> np.all(obj1 == (a == 1)) or obj1
+    True
+
+    >>> a = np.zeros(10, dtype=np.double)
+    >>> a0, obj0, a1, obj1 = test_broadcast_comparison(a)
+    >>> np.all(a0 == (a == 0)) or a0
+    True
+    >>> np.all(a1 == (a == 1)) or a1
+    True
+    >>> np.all(obj0 == (a == 0)) or obj0
+    True
+    >>> np.all(obj1 == (a == 1)) or obj1
+    True
+    """
+    cdef object obj = a
+    return a == 0, obj == 0, a == 1, obj == 1
+
+
+@testcase
+def test_c_api_searchsorted(np.ndarray arr, other):
+    """
+    >>> arr = np.random.randn(10)
+    >>> other = np.random.randn(5)
+    >>> result, expected = test_c_api_searchsorted(arr, other)
+    >>> (result == expected).all()
+    True
+    """
+    result = np.PyArray_SearchSorted(arr, other, np.NPY_SEARCHRIGHT, NULL)
+
+    expected = arr.searchsorted(other, side="right")
+    return result, expected
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/numpy_ValueError_T172.pyx cython-0.20.1+1~202203241016-9537/tests/run/numpy_ValueError_T172.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/numpy_ValueError_T172.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/numpy_ValueError_T172.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 172
+# ticket: t172
 # tag: numpy
 
 __doc__ = u"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/or.pyx cython-0.20.1+1~202203241016-9537/tests/run/or.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/or.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/or.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -58,3 +58,18 @@
     5
     """
     return False or 5
+
+cdef class A(object):
+    def __repr__(self):
+        return "A"
+
+def test_GH2059_missing_cast():
+    """
+    >>> test_GH2059_missing_cast()
+    (A, A)
+    """
+    cdef A a = A()
+    cdef object o = None
+    cdef A a_first = a or o
+    cdef A a_second = o or a
+    return a_first, a_second
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/overflow_check.pxi cython-0.20.1+1~202203241016-9537/tests/run/overflow_check.pxi
--- cython-0.20.1+1~201611251650-6686/tests/run/overflow_check.pxi	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/overflow_check.pxi	2022-03-24 10:16:46.000000000 +0000
@@ -1,14 +1,15 @@
 cimport cython
 
 cdef object two = 2
-cdef int size_in_bits = sizeof(INT) * 8
 
+cdef int size_in_bits_ = sizeof(INT) * 8
 cdef bint is_signed_ = not ((-1) > 0)
-cdef INT max_value_ = (two ** (size_in_bits - is_signed_) - 1)
+cdef INT max_value_ = (two ** (size_in_bits_ - is_signed_) - 1)
 cdef INT min_value_ = ~max_value_
 cdef INT half_ = max_value_ // 2
 
 # Python visible.
+size_in_bits = size_in_bits_
 is_signed = is_signed_
 max_value = max_value_
 min_value = min_value_
@@ -30,9 +31,9 @@
         op_res = op(a, b)
     except OverflowError:
         assign_overflow = True
-    assert func_overflow == assign_overflow, "Inconsistant overflow: %s(%s, %s)" % (func, a, b)
+    assert func_overflow == assign_overflow, "Inconsistent overflow: %s(%s, %s)" % (func, a, b)
     if not func_overflow:
-        assert res == op_res, "Inconsistant values: %s(%s, %s) == %s != %s" % (func, a, b, res, op_res)
+        assert res == op_res, "Inconsistent values: %s(%s, %s) == %s != %s" % (func, a, b, res, op_res)
 
 medium_values = (max_value_ / 2, max_value_ / 3, min_value_ / 2, sqrt(max_value_) - 1, sqrt(max_value_) + 1)
 def run_test(func, op):
@@ -230,6 +231,17 @@
     """
     >>> test_lshift(1, 10)
     1024
+    >>> test_lshift(1, size_in_bits - 2) == 1 << (size_in_bits - 2)
+    True
+    >>> test_lshift(0, size_in_bits - 1)
+    0
+    >>> test_lshift(1, size_in_bits - 1) == 1 << (size_in_bits - 1) if not is_signed else True
+    True
+    >>> if is_signed: expect_overflow(test_lshift, 1, size_in_bits - 1)
+    >>> expect_overflow(test_lshift, 0, size_in_bits)
+    >>> expect_overflow(test_lshift, 1, size_in_bits)
+    >>> expect_overflow(test_lshift, 0, size_in_bits + 1)
+    >>> expect_overflow(test_lshift, 1, size_in_bits + 1)
     >>> expect_overflow(test_lshift, 1, 100)
     >>> expect_overflow(test_lshift, max_value, 1)
     >>> test_lshift(max_value, 0) == max_value
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/packedstruct_T290.pyx cython-0.20.1+1~202203241016-9537/tests/run/packedstruct_T290.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/packedstruct_T290.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/packedstruct_T290.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 290
+# ticket: t290
 
 """
 >>> f()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/parallel.pyx cython-0.20.1+1~202203241016-9537/tests/run/parallel.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/parallel.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/parallel.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -8,6 +8,9 @@
 
 openmp.omp_set_nested(1)
 
+cdef int forward(int x) nogil:
+    return x
+
 def test_parallel():
     """
     >>> test_parallel()
@@ -20,6 +23,9 @@
 
     with nogil, cython.parallel.parallel():
         buf[threadid()] = threadid()
+        # Recognise threadid() also when it's used in a function argument.
+        # See https://github.com/cython/cython/issues/3594
+        buf[forward(cython.parallel.threadid())] = forward(threadid())
 
     for i in range(maxthreads):
         assert buf[i] == i
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/parallel_swap_assign_T425.pyx cython-0.20.1+1~202203241016-9537/tests/run/parallel_swap_assign_T425.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/parallel_swap_assign_T425.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/parallel_swap_assign_T425.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 425
+# ticket: t425
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/partial_circular_import.srctree cython-0.20.1+1~202203241016-9537/tests/run/partial_circular_import.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/partial_circular_import.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/partial_circular_import.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,40 @@
+PYTHON -c 'if __import__("sys").version_info >= (3,7): import pkg.A'
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import pkg.A"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(
+    ext_modules = cythonize("*/*.py"),
+    )
+
+######## pkg/__init__.py ########
+
+######## pkg/A.py ########
+from . import B
+
+def verify(rel_B):
+    import pkg.B as abs_B
+    assert abs_B == rel_B
+verify(B)
+
+######## pkg/B.py ########
+from . import C
+
+def verify(rel_C):
+    import pkg.C as abs_C
+    assert abs_C == rel_C
+verify(C)
+
+######## pkg/C.py ########
+from . import B
+
+def verify(rel_B):
+    import pkg.B as abs_B
+    assert abs_B == rel_B
+
+verify(B)
+
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep442_tp_finalize.pyx cython-0.20.1+1~202203241016-9537/tests/run/pep442_tp_finalize.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pep442_tp_finalize.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep442_tp_finalize.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,303 @@
+# mode: run
+
+import gc
+
+cdef class nontrivial_del:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+
+def test_del():
+    """
+    >>> test_del()
+    start
+    init
+    del
+    finish
+    """
+    print("start")
+    d = nontrivial_del()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+cdef class del_and_dealloc:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_del_and_dealloc():
+    """
+    >>> test_del_and_dealloc()
+    start
+    init
+    del
+    dealloc
+    finish
+    """
+    print("start")
+    d = del_and_dealloc()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+cdef class del_with_exception:
+    def __init__(self):
+        print("init")
+
+    def __del__(self):
+        print("del")
+        raise Exception("Error")
+
+def test_del_with_exception():
+    """
+    >>> test_del_with_exception()
+    start
+    init
+    del
+    finish
+    """
+    print("start")
+    d = nontrivial_del()
+    d = None
+    gc.collect()
+    print("finish")
+
+
+def test_nontrivial_del_with_exception():
+    """
+    >>> test_nontrivial_del_with_exception()
+    start
+    init
+    del
+    end
+    """
+    print("start")
+    def inner():
+        c = nontrivial_del()
+        raise RuntimeError()
+
+    try:
+        inner()
+    except RuntimeError:
+        pass
+
+    print("end")
+
+
+cdef class parent:
+    def __del__(self):
+        print("del parent")
+
+class child(parent):
+    def __del__(self):
+        print("del child")
+
+def test_del_inheritance():
+    """
+    >>> test_del_inheritance()
+    start
+    del child
+    finish
+    """
+    print("start")
+    c = child()
+    c = None
+    gc.collect()
+    print("finish")
+
+
+cdef class cy_parent:
+    def __del__(self):
+        print("del cy_parent")
+
+    def __dealloc__(self):
+        print("dealloc cy_parent")
+
+class py_parent:
+    def __del__(self):
+        print("del py_parent")
+
+class multi_child(cy_parent, py_parent):
+    def __del__(self):
+        print("del child")
+
+def test_multiple_inheritance():
+    """
+    >>> test_multiple_inheritance()
+    start
+    del child
+    dealloc cy_parent
+    finish
+    """
+    print("start")
+    c = multi_child()
+    c = None
+    gc.collect()
+    print("finish")
+
+
+cdef class zombie_object:
+    def __del__(self):
+        global global_zombie_object
+        print("del")
+        global_zombie_object = self
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_zombie_object():
+    """
+    >>> test_zombie_object()
+    start
+    del
+    del global
+    del
+    finish
+    """
+    global global_zombie_object
+    print("start")
+    i = zombie_object()
+    i = None
+    print("del global")
+    del global_zombie_object
+    gc.collect()
+    print("finish")
+
+
+# Same as above, but the member
+# makes the class GC, so it
+# is deallocated
+cdef class gc_zombie_object:
+    cdef object x
+
+    def __del__(self):
+        global global_gc_zombie_object
+        print("del")
+        global_gc_zombie_object = self
+
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_gc_zombie_object():
+    """
+    >>> test_gc_zombie_object()
+    start
+    del
+    del global
+    dealloc
+    finish
+    """
+    global global_gc_zombie_object
+    print("start")
+    i = gc_zombie_object()
+    i = None
+    print("del global")
+    del global_gc_zombie_object
+    gc.collect()
+    print("finish")
+
+
+cdef class cdef_parent:
+    pass
+
+cdef class cdef_child(cdef_parent):
+    def __del__(self):
+        print("del")
+    def __dealloc__(self):
+        print("dealloc")
+
+def test_cdef_parent_object():
+    """
+    >>> test_cdef_parent_object()
+    start
+    del
+    dealloc
+    finish
+    """
+    print("start")
+    i = cdef_child()
+    i = None
+    gc.collect()
+    print("finish")
+
+
+cdef class cdef_nontrivial_parent:
+    def __del__(self):
+        print("del parent")
+    def __dealloc__(self):
+        print("dealloc parent")
+
+cdef class cdef_nontrivial_child(cdef_nontrivial_parent):
+    def __del__(self):
+        print("del child")
+    def __dealloc__(self):
+        print("dealloc child")
+
+def test_cdef_nontrivial_parent_object():
+    """
+    >>> test_cdef_nontrivial_parent_object()
+    start
+    del child
+    dealloc child
+    dealloc parent
+    finish
+    """
+    print("start")
+    i = cdef_nontrivial_child()
+    i = None
+    gc.collect()
+    print("finish")
+
+
+class python_child(cdef_nontrivial_parent):
+    def __del__(self):
+        print("del python child")
+        super().__del__()
+
+def test_python_child_object():
+    """
+    >>> test_python_child_object()
+    Traceback (most recent call last):
+    ...
+    RuntimeError: End function
+    """
+
+    def func(tp):
+        inst = tp()
+        raise RuntimeError("End function")
+
+    func(python_child)
+
+def test_python_child_fancy_inherit():
+    """
+    >>> test_python_child_fancy_inherit()
+    Traceback (most recent call last):
+    ...
+    RuntimeError: End function
+    """
+
+    # inherit using "true python" rather than Cython
+    globs = { 'cdef_nontrivial_parent': cdef_nontrivial_parent }
+
+    exec("""
+class derived_python_child(cdef_nontrivial_parent):
+    pass
+""", globs)
+
+    derived_python_child = globs['derived_python_child']
+
+    def func(tp):
+        inst = tp()
+        raise RuntimeError("End function")
+
+    func(derived_python_child)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep448_extended_unpacking.pyx cython-0.20.1+1~202203241016-9537/tests/run/pep448_extended_unpacking.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pep448_extended_unpacking.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep448_extended_unpacking.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,5 @@
+# mode: run
+# tag: all_language_levels
 
 cimport cython
 
@@ -144,6 +146,22 @@
     return (*a, *b, 2, *c)
 
 
+def unpack_tuple_in_string_formatting(a, *args):
+    """
+    >>> print(unpack_tuple_in_string_formatting(1, 2))
+    1 2
+    >>> print(unpack_tuple_in_string_formatting(1, 'x'))
+    1 'x'
+    >>> unpack_tuple_in_string_formatting(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...format...
+    >>> unpack_tuple_in_string_formatting(1, 2, 3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...format...
+    """
+    return "%s %r" % (a, *args)
+
+
 #### lists
 
 
@@ -254,6 +272,24 @@
     return [*a, *b, 2, *c]
 
 
+def unpack_starred_arg_for_in_operator(x, l, m):
+    """
+    >>> l = [1,2,3]
+    >>> m = [4,5,6]
+    >>> x = 1
+    >>> unpack_starred_arg_for_in_operator(x, l, m)
+    True
+    >>> x = 10
+    >>> unpack_starred_arg_for_in_operator(x, l, m)
+    False
+    >>> unpack_starred_arg_for_in_operator(x, l, [])
+    False
+    >>> unpack_starred_arg_for_in_operator(x, [], [])
+    False
+    """
+    return x in [*l, *m]
+
+
 ###### sets
 
 
@@ -446,6 +482,10 @@
     return {**it}
 
 
+@cython.test_assert_path_exists('//MergedDictNode')
+@cython.test_fail_if_path_exists(
+    '//MergedDictNode//MergedDictNode',
+)
 def unpack_dict_from_iterable(it):
     """
     >>> d = unpack_dict_from_iterable(dict(a=1, b=2, c=3))
@@ -518,3 +558,28 @@
     True
     """
     return {**a, **b, 2: 4, **c}
+
+
+@cython.test_assert_path_exists(
+    '//MergedDictNode',
+    '//MergedDictNode//MergedDictNode',
+    '//MergedDictNode//MergedDictNode//DictNode',
+)
+def unpack_in_call(f):
+    """
+    >>> def f(a=1, test=2, **kwargs):
+    ...     return a, test, sorted(kwargs.items())
+    >>> wrapped = unpack_in_call(f)
+    >>> wrapped(1)
+    (1, 1, [('more', 2)])
+    >>> wrapped(test='overwritten')
+    (1, 1, [('more', 2)])
+    >>> wrapped(b=3)
+    (1, 1, [('b', 3), ('more', 2)])
+    >>> wrapped(more=4)
+    Traceback (most recent call last):
+    TypeError: function() got multiple values for keyword argument 'more'
+    """
+    def wrapper(*args, **kwargs):
+        return f(*args, more=2, **{**kwargs, 'test': 1})
+    return wrapper
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep448_test_extcall.pyx cython-0.20.1+1~202203241016-9537/tests/run/pep448_test_extcall.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pep448_test_extcall.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep448_test_extcall.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -224,10 +224,10 @@
 
 def call_nonseq_positional1():
     """
-    >>> call_nonseq_positional1()
+    >>> call_nonseq_positional1()  # doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: 'Nothing' object is not iterable
+    TypeError: ...Nothing...
 
     # TypeError: g() argument after * must be a sequence, not Nothing
     """
@@ -237,10 +237,10 @@
 
 def call_nonseq_positional2():
     """
-    >>> call_nonseq_positional2()
+    >>> call_nonseq_positional2()  # doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: 'Nothing' object is not iterable
+    TypeError: ...Nothing...
 
     # TypeError: g() argument after * must be a sequence, not Nothing
     """
@@ -326,7 +326,7 @@
     """
     >>> errors_non_string_kwarg()  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: ...keywords must be strings
+    TypeError: ...keywords must be strings...
     """
     f(**{1:2})
 
@@ -463,10 +463,10 @@
 
 def call_builtin_nonempty_dict():
     """
-    >>> call_builtin_nonempty_dict()
+    >>> call_builtin_nonempty_dict() # doctest: +ELLIPSIS
     Traceback (most recent call last):
       ...
-    TypeError: id() takes no keyword arguments
+    TypeError: id() ... keyword argument...
     """
     return id(1, **{'foo': 1})
 
@@ -474,7 +474,7 @@
 ''' Cython: currently just passes empty kwargs into f() while CPython keeps the content
 
 # A corner case of keyword dictionary items being deleted during
-# the function call setup. See .
+# the function call setup. See .
 
 def call_kwargs_modified_while_building():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep526_variable_annotations_cy.pyx cython-0.20.1+1~202203241016-9537/tests/run/pep526_variable_annotations_cy.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pep526_variable_annotations_cy.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep526_variable_annotations_cy.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,58 @@
+# mode: run
+
+import cython
+
+try:
+    import typing
+    from typing import List, Tuple
+    from typing import Set as _SET_
+except:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    cdef typing.Dict[int, float] a = {}
+    cdef List[int] b = []
+    cdef _SET_[object] c = set()
+
+    print(cython.typeof(a))
+    print(cython.typeof(b))
+    print(cython.typeof(c))
+
+cdef class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    cdef int regular
+    cdef typing.ClassVar[int] cls
+    cls = 5
+
+# because tuple is specifically special cased to go to ctuple where possible
+def test_tuple(typing.Tuple[int, float] a,  typing.Tuple[int, ...] b,
+               Tuple[int, object] c  # cannot be a ctuple
+               ):
+    """
+    >>> test_tuple((1, 1.0), (1, 1.0), (1, 1.0))
+    int
+    int
+    tuple object
+    tuple object
+    """
+    cdef typing.Tuple[int, float] x = (a[0], a[1])
+    cdef Tuple[int, ...] y = (1,2.)
+    z = a[0]  # should infer to int
+
+    print(cython.typeof(z))
+    print(cython.typeof(x[0]))
+    print(cython.typeof(y))
+    print(cython.typeof(c))
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep526_variable_annotations.py cython-0.20.1+1~202203241016-9537/tests/run/pep526_variable_annotations.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pep526_variable_annotations.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep526_variable_annotations.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,215 @@
+# cython: language_level=3
+# mode: run
+# tag: pure3.6, pep526, pep484, warnings
+
+import cython
+
+from typing import Dict, List, TypeVar, Optional, Generic, Tuple
+
+try:
+    import typing
+    from typing import Set as _SET_
+    from typing import ClassVar
+except ImportError:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+
+var = 1  # type: annotation
+var: int = 2
+fvar: float = 1.2
+some_number: cython.int    # variable without initial value
+some_list: List[int] = []  # variable with initial value
+t: Tuple[int, ...] = (1, 2, 3)
+body: Optional[List[str]]
+descr_only : "descriptions are allowed but ignored"
+
+
+some_number = 5
+body = None
+
+
+def f():
+    """
+    >>> f()
+    (2, 1.5, [], (1, 2, 3))
+    """
+    var = 1  # type: annotation
+    var: int = 2
+    fvar: float = 1.5
+    some_number: cython.int    # variable without initial value
+    some_list: List[int] = []  # variable with initial value
+    t: Tuple[int, ...] = (1, 2, 3)
+    body: Optional[List[str]]
+    descr_only: "descriptions are allowed but ignored"
+
+    return var, fvar, some_list, t
+
+
+class BasicStarship(object):
+    """
+    >>> bs = BasicStarship(5)
+    >>> bs.damage
+    5
+    >>> bs.captain
+    'Picard'
+    >>> bs.stats
+    {}
+    >>> BasicStarship.stats
+    {}
+    """
+    captain: str = 'Picard'               # instance variable with default
+    damage: cython.int                    # instance variable without default
+    stats: ClassVar[Dict[str, int]] = {}  # class variable
+    descr_only: "descriptions are allowed but ignored"
+
+    def __init__(self, damage):
+        self.damage = damage
+
+
+@cython.cclass
+class BasicStarshipExt(object):
+    """
+    >>> bs = BasicStarshipExt(5)
+    >>> bs.test()
+    (5, 'Picard', {})
+    """
+    captain: str = 'Picard'               # instance variable with default
+    damage: cython.int                    # instance variable without default
+    stats: ClassVar[Dict[str, int]] = {}  # class variable
+    descr_only: "descriptions are allowed but ignored"
+
+    def __init__(self, damage):
+        self.damage = damage
+
+    def test(self):
+        return self.damage, self.captain, self.stats
+
+
+T = TypeVar('T')
+
+
+# FIXME: this fails in Py3.7 now
+#class Box(Generic[T]):
+#    def __init__(self, content):
+#        self.content: T = content
+#
+#box = Box(content=5)
+
+
+class Cls(object):
+    pass
+
+
+c = Cls()
+c.x: int = 0  # Annotates c.x with int.
+c.y: int      # Annotates c.y with int.
+
+d = {}
+d['a']: int = 0  # Annotates d['a'] with int.
+d['b']: int      # Annotates d['b'] with int.
+
+(x): int      # Annotates x with int, (x) treated as expression by compiler.
+(y): int = 0  # Same situation here.
+
+
+@cython.test_assert_path_exists(
+    "//WhileStatNode",
+    "//WhileStatNode//DictIterationNextNode",
+)
+def iter_declared_dict(d):
+    """
+    >>> d = {1.1: 2.5, 3.3: 4.5}
+    >>> iter_declared_dict(d)
+    7.0
+
+    # specialized "compiled" test in module-level __doc__
+    """
+    typed_dict : Dict[float, float] = d
+    s = 0.0
+    for key in typed_dict:
+        s += d[key]
+    return s
+
+
+@cython.test_assert_path_exists(
+    "//WhileStatNode",
+    "//WhileStatNode//DictIterationNextNode",
+)
+def iter_declared_dict_arg(d : Dict[float, float]):
+    """
+    >>> d = {1.1: 2.5, 3.3: 4.5}
+    >>> iter_declared_dict_arg(d)
+    7.0
+
+    # module level "compiled" test in __doc__ below
+    """
+    s = 0.0
+    for key in d:
+        s += d[key]
+    return s
+
+
+def literal_list_ptr():
+    """
+    >>> literal_list_ptr()
+    4
+    """
+    a : cython.p_int = [1, 2, 3, 4, 5]
+    return a[3]
+
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    a: typing.Dict[int, float] = {}
+    b: List[int] = []
+    c: _SET_[object] = set()
+
+    print(cython.typeof(a) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(b) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+# because tuple is specifically special cased to go to ctuple where possible
+def test_tuple(a: typing.Tuple[int, float], b: typing.Tuple[int, ...],
+               c: Tuple[int, object]  # cannot be a ctuple
+               ):
+    """
+    >>> test_tuple((1, 1.0), (1, 1.0), (1, 1.0))
+    int
+    int
+    tuple object
+    tuple object
+    """
+    x: typing.Tuple[int, float] = (a[0], a[1])
+    y: Tuple[int, ...] = (1,2.)
+    z = a[0]  # should infer to int
+
+    print(cython.typeof(z))
+    print(cython.typeof(x[0]))
+    print(cython.typeof(y) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+
+if cython.compiled:
+    __doc__ = """
+    # passing non-dicts to variables declared as dict now fails
+    >>> class D(object):
+    ...     def __getitem__(self, x): return 2
+    ...     def __iter__(self): return iter([1, 2, 3])
+    >>> iter_declared_dict(D())  # doctest:+IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ...
+    TypeError: Expected dict, got D
+    >>> iter_declared_dict_arg(D())  # doctest:+IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+    ...
+    TypeError: Expected dict, got D
+    """
+
+
+_WARNINGS = """
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep557_dataclasses.py cython-0.20.1+1~202203241016-9537/tests/run/pep557_dataclasses.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pep557_dataclasses.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep557_dataclasses.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,54 @@
+# mode: run
+# tag: pep557, pure3.7
+
+import dataclasses
+from typing import Sequence
+
+
+@dataclasses.dataclass
+class Color:
+    """
+    >>> list(Color.__dataclass_fields__.keys())
+    ['red', 'green', 'blue', 'alpha']
+    >>> Color(1, 2, 3)
+    Color(red=1, green=2, blue=3, alpha=255)
+    >>> Color(1, 2, 3, 4)
+    Color(red=1, green=2, blue=3, alpha=4)
+    >>> Color(green=1, blue=2, red=3, alpha=40)
+    Color(red=3, green=1, blue=2, alpha=40)
+    """
+    red: int
+    green: int
+    blue: int
+    alpha: int = 255
+
+
+@dataclasses.dataclass
+class NamedColor(Color):
+    """
+    >>> list(NamedColor.__dataclass_fields__.keys())
+    ['red', 'green', 'blue', 'alpha', 'names']
+    >>> NamedColor(1, 2, 3)
+    NamedColor(red=1, green=2, blue=3, alpha=255, names=[])
+    >>> NamedColor(1, 2, 3, 4)
+    NamedColor(red=1, green=2, blue=3, alpha=4, names=[])
+    >>> NamedColor(green=1, blue=2, red=3, alpha=40)
+    NamedColor(red=3, green=1, blue=2, alpha=40, names=[])
+    >>> NamedColor(1, 2, 3, names=["blackish", "very dark cyan"])
+    NamedColor(red=1, green=2, blue=3, alpha=255, names=['blackish', 'very dark cyan'])
+    """
+    names: Sequence[str] = dataclasses.field(default_factory=list)
+
+
+@dataclasses.dataclass(frozen=True)
+class IceCream:
+    """
+    >>> IceCream("vanilla")
+    IceCream(flavour='vanilla', num_toppings=2)
+    >>> IceCream("vanilla") == IceCream("vanilla", num_toppings=3)
+    False
+    >>> IceCream("vanilla") == IceCream("vanilla", num_toppings=2)
+    True
+    """
+    flavour: str
+    num_toppings: int = 2
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pep563_annotations.py cython-0.20.1+1~202203241016-9537/tests/run/pep563_annotations.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pep563_annotations.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pep563_annotations.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,40 @@
+# mode: run
+# tag: pep563, pure3.7
+
+from __future__ import annotations
+
+def f(a: 1+2==3, b: list, c: this_cant_evaluate, d: "Hello from inside a string") -> "Return me!":
+    """
+    The absolute exact strings aren't reproducible according to the PEP,
+    so be careful to avoid being too specific
+    >>> stypes = (type(""), type(u"")) # Python 2 is a bit awkward here
+    >>> eval(f.__annotations__['a'])
+    True
+    >>> isinstance(f.__annotations__['a'], stypes)
+    True
+    >>> print(f.__annotations__['b'])
+    list
+    >>> print(f.__annotations__['c'])
+    this_cant_evaluate
+    >>> isinstance(eval(f.__annotations__['d']), stypes)
+    True
+    >>> print(f.__annotations__['return'][1:-1]) # First and last could be either " or '
+    Return me!
+    >>> f.__annotations__['return'][0] == f.__annotations__['return'][-1]
+    True
+    """
+    pass
+
+
+def empty_decorator(cls):
+    return cls
+
+
+@empty_decorator
+class DecoratedStarship(object):
+    """
+    >>> sorted(DecoratedStarship.__annotations__.items())
+    [('captain', 'str'), ('damage', 'cython.int')]
+    """
+    captain: str = 'Picard'               # instance variable with default
+    damage: cython.int                    # instance variable without default
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/posix_time.pyx cython-0.20.1+1~202203241016-9537/tests/run/posix_time.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/posix_time.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/posix_time.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -22,7 +22,7 @@
     t.it_value.tv_sec = 0
     t.it_value.tv_usec = 0
     ret = setitimer(ITIMER_REAL, &t, NULL)
-    return gtime.it_interval.tv_sec, gtime.it_interval.tv_usec
+    return int(gtime.it_interval.tv_sec), int(gtime.it_interval.tv_usec)
 
 def test_gettimeofday():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/posonly.py cython-0.20.1+1~202203241016-9537/tests/run/posonly.py
--- cython-0.20.1+1~201611251650-6686/tests/run/posonly.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/posonly.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,568 @@
+# cython: always_allow_keywords=True
+# mode: run
+# tag: posonly, pure3.8
+
+import cython
+import sys
+import pickle
+
+def test_optional_posonly_args1(a, b=10, /, c=100):
+    """
+    >>> test_optional_posonly_args1(1, 2, 3)
+    6
+    >>> test_optional_posonly_args1(1, 2, c=3)
+    6
+    >>> test_optional_posonly_args1(1, b=2, c=3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args1() got ... keyword argument... 'b'
+    >>> test_optional_posonly_args1(1, 2)
+    103
+    >>> test_optional_posonly_args1(1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args1() got ... keyword argument... 'b'
+    """
+    return a + b + c
+
+def test_optional_posonly_args2(a=1, b=10, /, c=100):
+    """
+    >>> test_optional_posonly_args2(1, 2, 3)
+    6
+    >>> test_optional_posonly_args2(1, 2, c=3)
+    6
+    >>> test_optional_posonly_args2(1, b=2, c=3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args2() got ... keyword argument... 'b'
+    >>> test_optional_posonly_args2(1, 2)
+    103
+    >>> test_optional_posonly_args2(1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_optional_posonly_args2() got ... keyword argument... 'b'
+    >>> test_optional_posonly_args2(1, c=2)
+    13
+    """
+    return a + b + c
+
+# TODO: this causes a line that is too long for old versions of Clang
+#def many_args(a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,
+#              a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40,
+#              a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59,
+#              a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78,
+#              a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91,a92,a93,a94,a95,a96,a97,
+#              a98,a99,a100,a101,a102,a103,a104,a105,a106,a107,a108,a109,a110,a111,a112,
+#              a113,a114,a115,a116,a117,a118,a119,a120,a121,a122,a123,a124,a125,a126,a127,
+#              a128,a129,a130,a131,a132,a133,a134,a135,a136,a137,a138,a139,a140,a141,a142,
+#              a143,a144,a145,a146,a147,a148,a149,a150,a151,a152,a153,a154,a155,a156,a157,
+#              a158,a159,a160,a161,a162,a163,a164,a165,a166,a167,a168,a169,a170,a171,a172,
+#              a173,a174,a175,a176,a177,a178,a179,a180,a181,a182,a183,a184,a185,a186,a187,
+#              a188,a189,a190,a191,a192,a193,a194,a195,a196,a197,a198,a199,a200,a201,a202,
+#              a203,a204,a205,a206,a207,a208,a209,a210,a211,a212,a213,a214,a215,a216,a217,
+#              a218,a219,a220,a221,a222,a223,a224,a225,a226,a227,a228,a229,a230,a231,a232,
+#              a233,a234,a235,a236,a237,a238,a239,a240,a241,a242,a243,a244,a245,a246,a247,
+#              a248,a249,a250,a251,a252,a253,a254,a255,a256,a257,a258,a259,a260,a261,a262,
+#              a263,a264,a265,a266,a267,a268,a269,a270,a271,a272,a273,a274,a275,a276,a277,
+#              a278,a279,a280,a281,a282,a283,a284,a285,a286,a287,a288,a289,a290,a291,a292,
+#              a293,a294,a295,a296,a297,a298,a299,/,b,c=42,*,d):
+#    """
+#    >>> many_args(*range(299),b=1,c=2,d=3)
+#    (298, 1, 2, 3)
+#    >>> many_args(*range(299),b=1,d=3)
+#    (298, 1, 42, 3)
+#    >>> many_args(*range(300),d=3)
+#    (298, 299, 42, 3)
+#    """
+#    return (a299, b, c, d)
+
+#TODO: update this test for Python 3.8 final
+@cython.binding(True)
+def func_introspection1(a, b, c, /, d, e=1, *, f, g=2):
+    """
+    >>> if sys.version_info[0] < 3:
+    ...     assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount
+    ... else:
+    ...     assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount
+    >>> func_introspection1.__defaults__
+    (1,)
+    """
+
+@cython.binding(True)
+def func_introspection2(a, b, c=1, /, d=2, e=3, *, f, g=4):
+    """
+    >>> if sys.version_info[0] < 3:
+    ...     assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount
+    ... else:
+    ...     assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount
+    >>> func_introspection2.__defaults__
+    (1, 2, 3)
+    """
+
+def test_pos_only_call_via_unpacking(a, b, /):
+    """
+    >>> test_pos_only_call_via_unpacking(*[1,2])
+    3
+    """
+    return a + b
+
+def test_use_positional_as_keyword1(a, /):
+    """
+    >>> test_use_positional_as_keyword1(1)
+    >>> test_use_positional_as_keyword1(a=1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword1() ... keyword argument...
+    """
+
+def test_use_positional_as_keyword2(a, /, b):
+    """
+    >>> test_use_positional_as_keyword2(1, 2)
+    >>> test_use_positional_as_keyword2(1, b=2)
+    >>> test_use_positional_as_keyword2(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword2() ... positional...argument...
+    """
+
+def test_use_positional_as_keyword3(a, b, /):
+    """
+    >>> test_use_positional_as_keyword3(1, 2)
+    >>> test_use_positional_as_keyword3(a=1, b=2) # doctest:+ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_use_positional_as_keyword3() got ... keyword argument...
+    """
+
+def test_positional_only_and_arg_invalid_calls(a, b, /, c):
+    """
+    >>> test_positional_only_and_arg_invalid_calls(1, 2, 3)
+    >>> test_positional_only_and_arg_invalid_calls(1, 2, c=3)
+    >>> test_positional_only_and_arg_invalid_calls(1, 2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() ... positional argument...
+    >>> test_positional_only_and_arg_invalid_calls(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() ... positional arguments...
+    >>> test_positional_only_and_arg_invalid_calls(1,2,3,4)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_arg_invalid_calls() takes ... positional arguments ...4 ...given...
+    """
+
+def test_positional_only_and_optional_arg_invalid_calls(a, b, /, c=3):
+    """
+    >>> test_positional_only_and_optional_arg_invalid_calls(1, 2)
+    >>> test_positional_only_and_optional_arg_invalid_calls(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional argument...
+    >>> test_positional_only_and_optional_arg_invalid_calls()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional arguments...
+    >>> test_positional_only_and_optional_arg_invalid_calls(1, 2, 3, 4)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_optional_arg_invalid_calls() takes ... positional arguments ...4 ...given...
+    """
+
+def test_positional_only_and_kwonlyargs_invalid_calls(a, b, /, c, *, d, e):
+    """
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=2)
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, e=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional argument...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, 4, 5, 6, d=7, e=8)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() takes ... positional arguments ...
+    >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=4, f=56)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_and_kwonlyargs_invalid_calls() got an unexpected keyword argument 'f'
+    """
+
+def test_positional_only_invalid_calls(a, b, /):
+    """
+    >>> test_positional_only_invalid_calls(1, 2)
+    >>> test_positional_only_invalid_calls(1)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() ... positional argument...
+    >>> test_positional_only_invalid_calls()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() ... positional arguments...
+    >>> test_positional_only_invalid_calls(1, 2, 3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_invalid_calls() takes ... positional arguments ...3 ...given...
+    """
+
+def test_positional_only_with_optional_invalid_calls(a, b=2, /):
+    """
+    >>> test_positional_only_with_optional_invalid_calls(1)
+    >>> test_positional_only_with_optional_invalid_calls()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_with_optional_invalid_calls() ... positional argument...
+    >>> test_positional_only_with_optional_invalid_calls(1, 2, 3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_positional_only_with_optional_invalid_calls() takes ... positional arguments ...3 ...given...
+    """
+
+def test_no_standard_args_usage(a, b, /, *, c):
+    """
+    >>> test_no_standard_args_usage(1, 2, c=3)
+    >>> test_no_standard_args_usage(1, b=2, c=3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_no_standard_args_usage() ... positional... argument...
+    """
+
+#def test_change_default_pos_only():
+# TODO: probably remove this, since  __defaults__ is not writable in Cython?
+#    """
+#    >>> test_change_default_pos_only()
+#    True
+#    True
+#    """
+#    def f(a, b=2, /, c=3):
+#        return a + b + c
+#
+#    print((2,3) == f.__defaults__)
+#    f.__defaults__ = (1, 2, 3)
+#    print(f(1, 2, 3) == 6)
+
+def test_lambdas():
+    """
+    >>> test_lambdas()
+    3
+    3
+    3
+    3
+    3
+    """
+    x = lambda a, /, b: a + b
+    print(x(1,2))
+    print(x(1,b=2))
+
+    x = lambda a, /, b=2: a + b
+    print(x(1))
+
+    x = lambda a, b, /: a + b
+    print(x(1, 2))
+
+    x = lambda a, b, /, : a + b
+    print(x(1, 2))
+
+class TestPosonlyMethods(object):
+    """
+    >>> TestPosonlyMethods().f(1,2)
+    (1, 2)
+    >>> TestPosonlyMethods.f(TestPosonlyMethods(), 1, 2)
+    (1, 2)
+    >>> try:
+    ...     TestPosonlyMethods.f(1,2)
+    ... except TypeError:
+    ...    print("Got type error")
+    Got type error
+    >>> TestPosonlyMethods().f(1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...f() got ... keyword argument... 'b'
+    """
+    def f(self, a, b, /):
+        return a, b
+
+class TestMangling(object):
+    """
+    >>> TestMangling().f()
+    42
+    >>> TestMangling().f2()
+    42
+
+    #>>> TestMangling().f3()
+    #(42, 43)
+    #>>> TestMangling().f4()
+    #(42, 43, 44)
+
+    >>> TestMangling().f2(1)
+    1
+
+    #>>> TestMangling().f3(1, _TestMangling__b=2)
+    #(1, 2)
+    #>>> TestMangling().f4(1, _TestMangling__b=2, _TestMangling__c=3)
+    #(1, 2, 3)
+    """
+    def f(self, *, __a=42):
+        return __a
+
+    def f2(self, __a=42, /):
+        return __a
+
+# FIXME: https://github.com/cython/cython/issues/1382
+#    def f3(self, __a=42, /, __b=43):
+#        return (__a, __b)
+
+#    def f4(self, __a=42, /, __b=43, *, __c=44):
+#        return (__a, __b, __c)
+
+def test_module_function(a, b, /):
+    """
+    >>> test_module_function(1, 2)
+    >>> test_module_function()  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_module_function() ... positional arguments...
+    """
+
+def test_closures1(x,y):
+    """
+    >>> test_closures1(1,2)(3,4)
+    10
+    >>> test_closures1(1,2)(3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...g() ... positional argument...
+    >>> test_closures1(1,2)(3,4,5)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...g() ... positional argument...
+    """
+    def g(x2, /, y2):
+        return x + y + x2 + y2
+    return g
+
+def test_closures2(x, /, y):
+    """
+    >>> test_closures2(1,2)(3,4)
+    10
+    """
+    def g(x2,y2):
+        return x + y + x2 + y2
+    return g
+
+
+def test_closures3(x, /, y):
+    """
+    >>> test_closures3(1,2)(3,4)
+    10
+    >>> test_closures3(1,2)(3)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...g() ... positional argument...
+    >>> test_closures3(1,2)(3,4,5)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...g() ... positional argument...
+    """
+    def g(x2, /, y2):
+        return x + y + x2 + y2
+    return g
+
+
+def test_same_keyword_as_positional_with_kwargs(something, /, **kwargs):
+    """
+    >>> test_same_keyword_as_positional_with_kwargs(42, something=42)
+    (42, {'something': 42})
+    >>> test_same_keyword_as_positional_with_kwargs(something=42)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_same_keyword_as_positional_with_kwargs() ... positional argument...
+    >>> test_same_keyword_as_positional_with_kwargs(42)
+    (42, {})
+    """
+    return (something, kwargs)
+
+def test_serialization1(a, b, /):
+    """
+    >>> pickled_posonly = pickle.dumps(test_serialization1)
+    >>> unpickled_posonly = pickle.loads(pickled_posonly)
+    >>> unpickled_posonly(1, 2)
+    (1, 2)
+    >>> unpickled_posonly(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_serialization1() got ... keyword argument...
+    """
+    return (a, b)
+
+def test_serialization2(a, /, b):
+    """
+    >>> pickled_optional = pickle.dumps(test_serialization2)
+    >>> unpickled_optional = pickle.loads(pickled_optional)
+    >>> unpickled_optional(1, 2)
+    (1, 2)
+    >>> unpickled_optional(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_serialization2() ... positional... argument...
+    """
+    return (a, b)
+
+def test_serialization3(a=1, /, b=2):
+    """
+    >>> pickled_defaults = pickle.dumps(test_serialization3)
+    >>> unpickled_defaults = pickle.loads(pickled_defaults)
+    >>> unpickled_defaults(1, 2)
+    (1, 2)
+    >>> unpickled_defaults(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_serialization3() got ... keyword argument... 'a'
+    """
+    return (a, b)
+
+
+async def test_async(a=1, /, b=2):
+    """
+    >>> test_async(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_async() got ... keyword argument... 'a'
+    """
+    return a, b
+
+
+def test_async_call(*args, **kwargs):
+    """
+    >>> test_async_call(1, 2)
+    >>> test_async_call(1, b=2)
+    >>> test_async_call(1)
+    >>> test_async_call()
+    """
+    if sys.version_info < (3, 6):
+        return
+    try:
+        coro = test_async(*args, **kwargs)
+        coro.send(None)
+    except StopIteration as e:
+        result = e.value
+    assert result == (1, 2), result
+
+
+def test_generator(a=1, /, b=2):
+    """
+    >>> test_generator(a=1, b=2)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: test_generator() got ... keyword argument... 'a'
+    >>> gen = test_generator(1, 2)
+    >>> next(gen)
+    (1, 2)
+    >>> gen = test_generator(1, b=2)
+    >>> next(gen)
+    (1, 2)
+    >>> gen = test_generator(1)
+    >>> next(gen)
+    (1, 2)
+    >>> gen = test_generator()
+    >>> next(gen)
+    (1, 2)
+    """
+    yield a, b
+
+def f_call_1_0_0(a,/):
+    """
+    >>> f_call_1_0_0(1)
+    (1,)
+    """
+    return (a,)
+
+def f_call_1_1_0(a, /, b):
+    """
+    >>> f_call_1_1_0(1,2)
+    (1, 2)
+    """
+    return (a,b)
+
+def f_call_1_1_1(a, /, b, *, c):
+    """
+    >>> f_call_1_1_1(1,2,c=3)
+    (1, 2, 3)
+    """
+    return (a,b,c)
+
+def f_call_1_1_1_star(a, /, b, *args, c):
+    """
+    >>> f_call_1_1_1_star(1,2,c=3)
+    (1, 2, (), 3)
+    >>> f_call_1_1_1_star(1,2,3,4,5,6,7,8,c=9)
+    (1, 2, (3, 4, 5, 6, 7, 8), 9)
+    """
+    return (a,b,args,c)
+
+def f_call_1_1_1_kwds(a, /, b, *, c, **kwds):
+    """
+    >>> f_call_1_1_1_kwds(1,2,c=3)
+    (1, 2, 3, {})
+    >>> f_call_1_1_1_kwds(1,2,c=3,d=4,e=5) == (1, 2, 3, {'d': 4, 'e': 5})
+    True
+    """
+    return (a,b,c,kwds)
+
+def f_call_1_1_1_star_kwds(a, /, b, *args, c, **kwds):
+    """
+    >>> f_call_1_1_1_star_kwds(1,2,c=3,d=4,e=5) == (1, 2, (), 3, {'d': 4, 'e': 5})
+    True
+    >>> f_call_1_1_1_star_kwds(1,2,3,4,c=5,d=6,e=7) == (1, 2, (3, 4), 5, {'d': 6, 'e': 7})
+    True
+    """
+    return (a,b,args,c,kwds)
+
+def f_call_one_optional_kwd(a, /, *, b=2):
+    """
+    >>> f_call_one_optional_kwd(1)
+    (1, 2)
+    >>> f_call_one_optional_kwd(1, b=3)
+    (1, 3)
+    """
+    return (a,b)
+
+def f_call_posonly_stararg(a, /, *args):
+    """
+    >>> f_call_posonly_stararg(1)
+    (1, ())
+    >>> f_call_posonly_stararg(1, 2, 3, 4)
+    (1, (2, 3, 4))
+    """
+    return (a,args)
+
+def f_call_posonly_kwarg(a, /, **kw):
+    """
+    >>> f_call_posonly_kwarg(1)
+    (1, {})
+    >>> all_args = f_call_posonly_kwarg(1, b=2, c=3, d=4)
+    >>> all_args == (1, {'b': 2, 'c': 3, 'd': 4}) or all_args
+    True
+    """
+    return (a,kw)
+
+def f_call_posonly_stararg_kwarg(a, /, *args, **kw):
+    """
+    >>> f_call_posonly_stararg_kwarg(1)
+    (1, (), {})
+    >>> f_call_posonly_stararg_kwarg(1, 2)
+    (1, (2,), {})
+    >>> all_args = f_call_posonly_stararg_kwarg(1, b=3, c=4)
+    >>> all_args == (1, (), {'b': 3, 'c': 4}) or all_args
+    True
+    >>> all_args = f_call_posonly_stararg_kwarg(1, 2, b=3, c=4)
+    >>> all_args == (1, (2,), {'b': 3, 'c': 4}) or all_args
+    True
+    """
+    return (a,args,kw)
+
+def test_empty_kwargs(a, b, /):
+    """
+    >>> test_empty_kwargs(1, 2)
+    (1, 2)
+    >>> test_empty_kwargs(1, 2, **{})
+    (1, 2)
+    >>> test_empty_kwargs(1, 2, **{'c': 3})
+    Traceback (most recent call last):
+    TypeError: test_empty_kwargs() got an unexpected keyword argument 'c'
+    """
+    return (a,b)
+
+
+@cython.cclass
+class TestExtensionClass:
+    """
+    >>> t = TestExtensionClass()
+    >>> t.f(1,2)
+    (1, 2, 3)
+    >>> t.f(1,2,4)
+    (1, 2, 4)
+    >>> t.f(1, 2, c=4)
+    (1, 2, 4)
+    >>> t.f(1, 2, 5, c=6)  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...f() got multiple values for ...argument 'c'
+    """
+    def f(self, a, b, /, c=3):
+        return (a,b,c)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/powop.pyx cython-0.20.1+1~202203241016-9537/tests/run/powop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/powop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/powop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -123,9 +123,9 @@
     0.5
     >>> optimised_pow2(0.5) == 2 ** 0.5
     True
-    >>> optimised_pow2('test')
+    >>> optimised_pow2('test') # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str'
+    TypeError: ...operand... **...
     """
     if isinstance(n, (int, long)) and 0 <= n < 1000:
         assert isinstance(2.0 ** n, float), 'float %s' % n
@@ -153,9 +153,9 @@
     0.5
     >>> optimised_pow2_inplace(0.5) == 2 ** 0.5
     True
-    >>> optimised_pow2_inplace('test')
+    >>> optimised_pow2_inplace('test') # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str'
+    TypeError: ...operand... **...
     """
     x = 2
     x **= n
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/property_decorator_T593.py cython-0.20.1+1~202203241016-9537/tests/run/property_decorator_T593.py
--- cython-0.20.1+1~201611251650-6686/tests/run/property_decorator_T593.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/property_decorator_T593.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 593
+# ticket: t593
 # tag: property, decorator
 
 my_property = property
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pstats_profile_test_pycfunc.pyx cython-0.20.1+1~202203241016-9537/tests/run/pstats_profile_test_pycfunc.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pstats_profile_test_pycfunc.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pstats_profile_test_pycfunc.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,228 @@
+# tag: pstats
+# cython: profile = True
+# cython: binding = False
+
+__doc__ = u"""
+    >>> import os, tempfile, cProfile as profile, pstats
+    >>> statsfile = tempfile.mkstemp()[1]
+    >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile)
+    >>> s = pstats.Stats(statsfile)
+    >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()])
+    >>> short_stats['f_def']
+    100
+    >>> short_stats['f_cdef']
+    100
+    >>> short_stats['f_cpdef']
+    200
+    >>> short_stats['f_cpdef (wrapper)']
+    100
+    >>> short_stats['f_inline']
+    100
+    >>> short_stats['f_inline_prof']
+    100
+    >>> short_stats['f_noprof']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'f_noprof'
+    >>> short_stats['f_raise']
+    100
+
+    >>> short_stats['withgil_prof']
+    100
+    >>> short_stats['withgil_noprof']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'withgil_noprof'
+
+    >>> short_stats['nogil_prof']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'nogil_prof'
+    >>> short_stats['nogil_noprof']
+    Traceback (most recent call last):
+    ...
+    KeyError: 'nogil_noprof'
+
+    >>> short_stats['f_raise']
+    100
+
+    >>> short_stats['m_def']
+    200
+    >>> short_stats['m_cdef']
+    100
+    >>> short_stats['m_cpdef']
+    200
+    >>> short_stats['m_cpdef (wrapper)']
+    100
+
+    >>> try:
+    ...    os.unlink(statsfile)
+    ... except:
+    ...    pass
+
+    >>> sorted(callees(s, 'test_profile'))  #doctest: +NORMALIZE_WHITESPACE
+    ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def',
+     'f_inline', 'f_inline_prof',
+     'f_raise',
+     'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def',
+     'withgil_prof']
+
+    >>> profile.runctx("test_generators()", locals(), globals(), statsfile)
+    >>> s = pstats.Stats(statsfile)
+    >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()])
+    >>> short_stats['generator']
+    3
+
+    >>> short_stats['generator_exception']
+    2
+
+    >>> short_stats['genexpr']
+    11
+
+    >>> sorted(callees(s, 'test_generators'))
+    ['call_generator', 'call_generator_exception', 'generator_expr']
+
+    >>> list(callees(s, 'call_generator'))
+    ['generator']
+
+    >>> list(callees(s, 'generator'))
+    []
+
+    >>> list(callees(s, 'generator_exception'))
+    []
+
+    >>> list(callees(s, 'generator_expr'))
+    ['genexpr']
+
+    >>> list(callees(s, 'genexpr'))
+    []
+
+    >>> def python_generator():
+    ...   yield 1
+    ...   yield 2
+    >>> def call_python_generator():
+    ...   list(python_generator())
+
+    >>> profile.runctx("call_python_generator()", locals(), globals(), statsfile)
+    >>> python_stats = pstats.Stats(statsfile)
+    >>> python_stats_dict = dict([(k[2], v[1]) for k,v in python_stats.stats.items()])
+
+    >>> profile.runctx("call_generator()", locals(), globals(), statsfile)
+    >>> cython_stats = pstats.Stats(statsfile)
+    >>> cython_stats_dict = dict([(k[2], v[1]) for k,v in cython_stats.stats.items()])
+
+    >>> python_stats_dict['python_generator'] == cython_stats_dict['generator']
+    True
+
+    >>> try:
+    ...    os.unlink(statsfile)
+    ... except:
+    ...    pass
+"""
+
+cimport cython
+
+def callees(pstats, target_caller):
+    pstats.calc_callees()
+    for (_, _, caller), callees in pstats.all_callees.items():
+      if caller == target_caller:
+        for (file, line, callee) in callees.keys():
+            if 'pyx' in file:
+                yield callee
+
+def test_profile(long N):
+    cdef long i, n = 0
+    cdef A a = A()
+    for i from 0 <= i < N:
+        n += f_def(i)
+        n += f_cdef(i)
+        n += f_cpdef(i)
+        n += (f_cpdef)(i)
+        n += f_inline(i)
+        n += f_inline_prof(i)
+        n += f_noprof(i)
+        n += nogil_noprof(i)
+        n += nogil_prof(i)
+        n += withgil_noprof(i)
+        n += withgil_prof(i)
+        n += a.m_def(i)
+        n += (a).m_def(i)
+        n += a.m_cpdef(i)
+        n += (a).m_cpdef(i)
+        n += a.m_cdef(i)
+        try:
+            n += f_raise(i+2)
+        except RuntimeError:
+            pass
+    return n
+
+def f_def(long a):
+    return a
+
+cdef long f_cdef(long a):
+    return a
+
+cpdef long f_cpdef(long a):
+    return a
+
+cdef inline long f_inline(long a):
+    return a
+
+@cython.profile(True)
+cdef inline long f_inline_prof(long a):
+    return a
+
+@cython.profile(False)
+cdef int f_noprof(long a):
+    return a
+
+cdef long f_raise(long) except -2:
+    raise RuntimeError
+
+@cython.profile(False)
+cdef int withgil_noprof(long a) with gil:
+    return (a)
+@cython.profile(True)
+cdef int withgil_prof(long a) with gil:
+    return (a)
+
+@cython.profile(False)
+cdef int nogil_noprof(long a) nogil:
+    return a
+@cython.profile(True)
+cdef int nogil_prof(long a) nogil:
+    return a
+
+cdef class A(object):
+    def m_def(self, long a):
+        return a
+    cpdef m_cpdef(self, long a):
+        return a
+    cdef m_cdef(self, long a):
+        return a
+
+def test_generators():
+    call_generator()
+    call_generator_exception()
+    generator_expr()
+
+def call_generator():
+    list(generator())
+
+def generator():
+    yield 1
+    yield 2
+
+def call_generator_exception():
+    try:
+        list(generator_exception())
+    except ValueError:
+        pass
+
+def generator_exception():
+    yield 1
+    raise ValueError(2)
+
+def generator_expr():
+    e = (x for x in range(10))
+    return sum(e)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pstats_profile_test.pyx cython-0.20.1+1~202203241016-9537/tests/run/pstats_profile_test.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pstats_profile_test.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pstats_profile_test.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,7 +1,7 @@
 # tag: pstats
 # cython: profile = True
 
-__doc__ = u"""
+u"""
     >>> import os, tempfile, cProfile as profile, pstats
     >>> statsfile = tempfile.mkstemp()[1]
     >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile)
@@ -12,9 +12,7 @@
     >>> short_stats['f_cdef']
     100
     >>> short_stats['f_cpdef']
-    200
-    >>> short_stats['f_cpdef (wrapper)']
-    100
+    300
     >>> short_stats['f_inline']
     100
     >>> short_stats['f_inline_prof']
@@ -49,10 +47,8 @@
     200
     >>> short_stats['m_cdef']
     100
-    >>> short_stats['m_cpdef']
-    200
-    >>> short_stats['m_cpdef (wrapper)']
-    100
+    >>> short_stats['m_cpdef'] - (200 if CPDEF_METHODS_COUNT_TWICE else 0)  # FIXME!
+    300
 
     >>> try:
     ...    os.unlink(statsfile)
@@ -60,15 +56,76 @@
     ...    pass
 
     >>> sorted(callees(s, 'test_profile'))  #doctest: +NORMALIZE_WHITESPACE
-    ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def',
+    ['f_cdef', 'f_cpdef', 'f_def',
      'f_inline', 'f_inline_prof',
      'f_raise',
-     'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def',
+     'm_cdef', 'm_cpdef', 'm_def',
      'withgil_prof']
+
+    >>> profile.runctx("test_generators()", locals(), globals(), statsfile)
+    >>> s = pstats.Stats(statsfile)
+    >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()])
+    >>> short_stats['generator']
+    3
+
+    >>> short_stats['generator_exception']
+    2
+
+    >>> short_stats['genexpr']
+    11
+
+    >>> sorted(callees(s, 'test_generators'))
+    ['call_generator', 'call_generator_exception', 'generator_expr']
+
+    >>> list(callees(s, 'call_generator'))
+    ['generator']
+
+    >>> list(callees(s, 'generator'))
+    []
+
+    >>> list(callees(s, 'generator_exception'))
+    []
+
+    >>> list(callees(s, 'generator_expr'))
+    ['genexpr']
+
+    >>> list(callees(s, 'genexpr'))
+    []
+
+    >>> def python_generator():
+    ...   yield 1
+    ...   yield 2
+    >>> def call_python_generator():
+    ...   list(python_generator())
+
+    >>> profile.runctx("call_python_generator()", locals(), globals(), statsfile)
+    >>> python_stats = pstats.Stats(statsfile)
+    >>> python_stats_dict = dict([(k[2], v[1]) for k,v in python_stats.stats.items()])
+
+    >>> profile.runctx("call_generator()", locals(), globals(), statsfile)
+    >>> cython_stats = pstats.Stats(statsfile)
+    >>> cython_stats_dict = dict([(k[2], v[1]) for k,v in cython_stats.stats.items()])
+
+    >>> python_stats_dict['python_generator'] == cython_stats_dict['generator']
+    True
+
+    >>> try:
+    ...    os.unlink(statsfile)
+    ... except:
+    ...    pass
 """
 
 cimport cython
 
+
+# FIXME: With type specs, cpdef methods are currently counted twice.
+# https://github.com/cython/cython/issues/2137
+cdef extern from *:
+    int CYTHON_USE_TYPE_SPECS
+
+CPDEF_METHODS_COUNT_TWICE = CYTHON_USE_TYPE_SPECS
+
+
 def callees(pstats, target_caller):
     pstats.calc_callees()
     for (_, _, caller), callees in pstats.all_callees.items():
@@ -77,10 +134,11 @@
             if 'pyx' in file:
                 yield callee
 
+
 def test_profile(long N):
     cdef long i, n = 0
     cdef A a = A()
-    for i from 0 <= i < N:
+    for i in range(N):
         n += f_def(i)
         n += f_cdef(i)
         n += f_cpdef(i)
@@ -147,3 +205,29 @@
         return a
     cdef m_cdef(self, long a):
         return a
+
+def test_generators():
+    call_generator()
+    call_generator_exception()
+    generator_expr()
+
+def call_generator():
+    list(generator())
+
+def generator():
+    yield 1
+    yield 2
+
+def call_generator_exception():
+    try:
+        list(generator_exception())
+    except ValueError:
+        pass
+
+def generator_exception():
+    yield 1
+    raise ValueError(2)
+
+def generator_expr():
+    e = (x for x in range(10))
+    return sum(e)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ptr_warning_T714.pyx cython-0.20.1+1~202203241016-9537/tests/run/ptr_warning_T714.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ptr_warning_T714.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ptr_warning_T714.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: werror
-# ticket: 714
+# ticket: t714
 
 def test_ptr():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/public_enum.pyx cython-0.20.1+1~202203241016-9537/tests/run/public_enum.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/public_enum.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/public_enum.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,10 +1,10 @@
-__doc__ = u"""
+# mode: run
+
+"""
 >>> BAR == 3
 True
 >>> HONK == 3+2+1
 True
->>> X == 4*5 + 1
-True
 >>> NONPUBLIC         # doctest: +ELLIPSIS
 Traceback (most recent call last):
 NameError: ...name 'NONPUBLIC' is not defined
@@ -12,8 +12,6 @@
 True
 """
 
-DEF X = 4*5
-
 cdef enum SECRET:
     NONPUBLIC = 23 + 42
 
@@ -21,4 +19,3 @@
     BAR = 3
     HONK = 3+2+1
     NOWPUBLIC = NONPUBLIC
-    X = X + 1          # FIXME: should this really work?
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/public_fused_types.srctree cython-0.20.1+1~202203241016-9537/tests/run/public_fused_types.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/public_fused_types.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/public_fused_types.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -196,8 +196,8 @@
 def ae(result, expected):
     "assert equals"
     if result != expected:
-        print 'result  :', result
-        print 'expected:', expected
+        print('result  :', result)
+        print('expected:', expected)
 
     assert result == expected
 
@@ -227,8 +227,7 @@
 
 d = {'obj': obj, 'myobj': myobj, 'ae': ae}
 
-# FIXME: uncomment after subclassing CyFunction
-#exec s in d
+exec(s, d)
 
 # Test def methods
 # ae(obj.def_method(12, 14.9), 26)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_dataclass.py cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_dataclass.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_dataclass.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_dataclass.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,31 @@
+# mode: run
+# tag: dataclass, pure3.7
+
+from __future__ import print_function
+
+import cython
+
+@cython.dataclasses.dataclass(order=True, unsafe_hash=True)
+@cython.cclass
+class MyDataclass:
+    """
+    >>> sorted(list(MyDataclass.__dataclass_fields__.keys()))
+    ['a', 'self']
+    >>> inst1 = MyDataclass(2.0, ['a', 'b'])
+    >>> print(inst1)
+    MyDataclass(a=2.0, self=['a', 'b'])
+    >>> inst2 = MyDataclass()
+    >>> print(inst2)
+    MyDataclass(a=1, self=[])
+    >>> inst1 == inst2
+    False
+    >>> inst1 > inst2
+    True
+    >>> inst2 == MyDataclass()
+    True
+    >>> hash(inst1) != id(inst1)
+    True
+    """
+
+    a: int = 1
+    self: list = cython.dataclasses.field(default_factory=list, hash=False)  # test that arguments of init don't conflict
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_property_decorator_T264.pxd cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_property_decorator_T264.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_property_decorator_T264.pxd	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_property_decorator_T264.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 264
+# ticket: t264
 # tag: property, decorator
 
 cdef class Prop:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_property_decorator_T264.py cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_property_decorator_T264.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_cdef_class_property_decorator_T264.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_cdef_class_property_decorator_T264.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 264
+# ticket: t264
 # tag: property, decorator
 
 class Prop(object):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/purecdef.py cython-0.20.1+1~202203241016-9537/tests/run/purecdef.py
--- cython-0.20.1+1~201611251650-6686/tests/run/purecdef.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/purecdef.py	2022-03-24 10:16:46.000000000 +0000
@@ -28,6 +28,17 @@
     def fwith2(a):
         return a*4
 
+    @cython.test_assert_path_exists(
+        '//CFuncDefNode',
+        '//LambdaNode',
+        '//GeneratorDefNode',
+        '//GeneratorBodyDefNode',
+    )
+    def f_with_genexpr(a):
+        f = lambda x: x+1
+        return (f(x) for x in a)
+
+
 with cclass:
     @cython.test_assert_path_exists('//CClassDefNode')
     class Egg(object):
@@ -123,3 +134,14 @@
     """
     x = cython.declare(int, 5)
     assert typed_return(cython.address(x))[0] is x
+
+
+def test_genexpr_in_cdef(l):
+    """
+    >>> gen = test_genexpr_in_cdef([1, 2, 3])
+    >>> list(gen)
+    [2, 3, 4]
+    >>> list(gen)
+    []
+    """
+    return f_with_genexpr(l)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_fused.pxd cython-0.20.1+1~202203241016-9537/tests/run/pure_fused.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_fused.pxd	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_fused.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,9 @@
+cimport cython
+
+ctypedef fused NotInPy:
+    int
+    float
+
+cdef class TestCls:
+    @cython.locals(loc = NotInPy)
+    cpdef cpfunc(self, NotInPy arg)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_fused.py cython-0.20.1+1~202203241016-9537/tests/run/pure_fused.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_fused.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_fused.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,64 @@
+# mode: run
+# tag: fused, pure3.6
+
+#cython: annotation_typing=True
+
+import cython
+
+InPy = cython.fused_type(cython.int, cython.float)
+
+class TestCls:
+    # although annotations as strings isn't recommended and generates a warning
+    # it does allow the test to run on more (pure) Python versions
+    def func1(self, arg: 'NotInPy'):
+        """
+        >>> TestCls().func1(1.0)
+        'float'
+        >>> TestCls().func1(2)
+        'int'
+        """
+        loc: 'NotInPy' = arg
+        return cython.typeof(arg)
+
+    if cython.compiled:
+        @cython.locals(arg=NotInPy, loc=NotInPy)  # NameError for 'NotInPy' in pure Python
+        def func2(self, arg):
+            """
+            >>> TestCls().func2(1.0)
+            'float'
+            >>> TestCls().func2(2)
+            'int'
+            """
+            loc = arg
+            return cython.typeof(arg)
+
+    def cpfunc(self, arg):
+        """
+        >>> TestCls().cpfunc(1.0)
+        'float'
+        >>> TestCls().cpfunc(2)
+        'int'
+        """
+        loc = arg
+        return cython.typeof(arg)
+
+    def func1_inpy(self, arg: InPy):
+        """
+        >>> TestCls().func1_inpy(1.0)
+        'float'
+        >>> TestCls().func1_inpy(2)
+        'int'
+        """
+        loc: InPy = arg
+        return cython.typeof(arg)
+
+    @cython.locals(arg = InPy, loc = InPy)
+    def func2_inpy(self, arg):
+        """
+        >>> TestCls().func2_inpy(1.0)
+        'float'
+        >>> TestCls().func2_inpy(2)
+        'int'
+        """
+        loc = arg
+        return cython.typeof(arg)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_parallel.py cython-0.20.1+1~202203241016-9537/tests/run/pure_parallel.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_parallel.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_parallel.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,60 @@
+# mode: run
+# tag: openmp, pure3.6
+
+import cython
+from cython.parallel import prange, parallel
+
+
+def prange_regression(n: cython.int, data: list):
+    """
+    >>> prange_regression(10, list(range(1, 4)))
+    19
+    """
+    s: cython.int = 0
+    i: cython.int
+    d: cython.int[3] = data
+
+    for i in prange(n, num_threads=3, nogil=True):
+        s += d[i % 3]
+    return s
+
+
+def prange_with_gil(n: cython.int, x):
+    """
+    >>> sum(3*i for i in range(10))
+    135
+    >>> prange_with_gil(10, 3)
+    135
+    """
+    i: cython.int
+    s: cython.int = 0
+
+    for i in prange(n, num_threads=3, nogil=True):
+        with cython.gil:
+            s += x * i
+
+    return s
+
+
+@cython.cfunc
+def use_nogil(x, i: cython.int) -> cython.int:
+    cx: cython.int = x
+    with cython.nogil:
+        return cx * i
+
+
+def prange_with_gil_call_nogil(n: cython.int, x):
+    """
+    >>> sum(3*i for i in range(10))
+    135
+    >>> prange_with_gil(10, 3)
+    135
+    """
+    i: cython.int
+    s: cython.int = 0
+
+    for i in prange(n, num_threads=3, nogil=True):
+        with cython.gil:
+            s += use_nogil(x, i)
+
+    return s
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_pxd.srctree cython-0.20.1+1~202203241016-9537/tests/run/pure_pxd.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_pxd.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_pxd.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 PYTHON setup.py build_ext --inplace
-PYTHON -c "import a; a.test()"
+PYTHON -c "import a; a.test(a)"
 
 ######## setup.py ########
 
@@ -37,7 +37,37 @@
         self.b = [1, 2, 3]
 
 
-def test():
+class TypedMethod():
+    """
+    >>> t = TypedMethod()
+    >>> t.meth()
+    97
+    """
+    def meth(self):
+        x = bytearray(b'abcdefg')
+        return x[0]
+
+
+def func(a, b, c):
+    """
+    >>> func(1, 2, 3)
+    6
+    """
+    return a + b + c
+
+def sum_generator_expression(a):
+    # GH-3477 - closure variables incorrectly captured in functions transformed to cdef
+    return sum(i for i in range(a))
+
+def run_sum_generator_expression(a):
+    """
+    >>> run_sum_generator_expression(5)
+    10
+    """
+    return sum_generator_expression(a)
+
+
+def test(module):
     import os.path
     assert not os.path.basename(__file__).endswith('.py'), __file__
     assert not os.path.basename(__file__).endswith('.pyc'), __file__
@@ -46,13 +76,17 @@
     assert not ExtTypePass().__doc__, ExtTypePass().__doc__
     assert ExtTypeDocstring().__doc__ == "huhu!", ExtTypeDocstring().__doc__
     assert ExtTypePxdDocstring().__doc__ == "ho, ho, ho!", ExtTypePxdDocstring().__doc__
+    assert '>>> ' in func.__doc__
 
     import doctest
-    doctest.testmod(verbose=True)
+    result = doctest.testmod(module, verbose=True)
+    assert not result.failed, result.failed
 
 
 ######## a.pxd ########
 
+cimport cython
+
 cdef class ExtTypePass:
     pass
 
@@ -64,3 +98,14 @@
 cdef class ExtTypeAttributes:
     cdef int a
     cdef readonly list b
+
+
+cdef class TypedMethod:
+    @cython.locals(x='char[:]')
+    cpdef meth(self)
+
+
+cpdef int func(x, int y, z) except? -1  # argument names should not matter, types should
+
+
+cdef int sum_generator_expression(int a)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_py3.py cython-0.20.1+1~202203241016-9537/tests/run/pure_py3.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_py3.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_py3.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,103 @@
+# mode: run
+# tag: annotation_typing, pure3.0, mypy
+
+import cython
+
+is_compiled = cython.compiled
+
+MyUnion = cython.union(n=cython.int, x=cython.double)
+MyStruct = cython.struct(is_integral=cython.bint, data=MyUnion)
+MyStruct2 = cython.typedef(MyStruct[2])  # type: cython.StructType
+
+
+@cython.ccall  # cpdef => C return type
+def test_return_type(n: cython.int) -> cython.double:
+    """
+    >>> test_return_type(389)
+    389.0
+    """
+    assert cython.typeof(n) == 'int', cython.typeof(n)
+    return n if is_compiled else float(n)
+
+
+def test_struct(n: cython.int, x: cython.double) -> MyStruct2:
+    """
+    >>> test_struct(389, 1.64493)
+    (389, 1.64493)
+    >>> d = test_struct.__annotations__
+    >>> sorted(d)
+    ['n', 'return', 'x']
+    """
+    assert cython.typeof(n) == 'int', cython.typeof(n)
+    if is_compiled:
+        assert cython.typeof(x) == 'double', cython.typeof(x)  # C double
+    else:
+        assert cython.typeof(x) == 'float', cython.typeof(x)   # Python float
+
+    a = cython.declare(MyStruct2)
+    a[0] = MyStruct(is_integral=True, data=MyUnion(n=n))
+    a[1] = MyStruct(is_integral=False, data={'x': x})
+    return a[0].data.n, a[1].data.x
+
+
+@cython.ccall
+def c_call(x) -> cython.double:
+    return x
+
+
+def call_ccall(x):
+    """
+    Test that a declared return type is honoured when compiled.
+
+    >>> result, return_type = call_ccall(1)
+
+    >>> (not is_compiled and 'double') or return_type
+    'double'
+    >>> (is_compiled and 'int') or return_type
+    'int'
+
+    >>> (not is_compiled and 1.0) or result
+    1.0
+    >>> (is_compiled and 1) or result
+    1
+    """
+    ret = c_call(x)
+    return ret, cython.typeof(ret)
+
+
+@cython.cfunc
+@cython.inline
+def cdef_inline(x) -> cython.double:
+    return x + 1
+
+
+def call_cdef_inline(x):
+    """
+    >>> result, return_type = call_cdef_inline(1)
+    >>> (not is_compiled and 'float') or type(result).__name__
+    'float'
+    >>> (not is_compiled and 'double') or return_type
+    'double'
+    >>> (is_compiled and 'int') or return_type
+    'int'
+    >>> result == 2.0  or  result
+    True
+    """
+    ret = cdef_inline(x)
+    return ret, cython.typeof(ret)
+
+@cython.cfunc
+def test_cdef_return_object(x: object) -> object:
+    """
+    Test support of python object in annotations
+    >>> test_cdef_return_object(3)
+    3
+    >>> test_cdef_return_object(None)
+    Traceback (most recent call last):
+        ...
+    RuntimeError
+    """
+    if x:
+        return x
+    else:
+        raise RuntimeError()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_py_cimports.py cython-0.20.1+1~202203241016-9537/tests/run/pure_py_cimports.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_py_cimports.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_py_cimports.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,13 @@
+# mode: run
+# tag: pure, import, cimport
+
+from cython.cimports.libc import math
+from cython.cimports.libc.math import ceil
+
+
+def libc_math_ceil(x):
+    """
+    >>> libc_math_ceil(1.5)
+    [2, 2]
+    """
+    return [int(n) for n in [ceil(x), math.ceil(x)]]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_py.py cython-0.20.1+1~202203241016-9537/tests/run/pure_py.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_py.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_py.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,8 @@
+import sys
+IS_PY2 = sys.version_info[0] < 3
+
 import cython
+from cython import sizeof
 
 is_compiled = cython.compiled
 
@@ -17,7 +21,7 @@
     """
     x = cython.declare(cython.bint)
     print(cython.sizeof(x) == cython.sizeof(cython.bint))
-    print(cython.sizeof(cython.char) <= cython.sizeof(cython.short) <= cython.sizeof(cython.int) <= cython.sizeof(cython.long) <= cython.sizeof(cython.longlong))
+    print(sizeof(cython.char) <= sizeof(cython.short) <= sizeof(cython.int) <= sizeof(cython.long) <= sizeof(cython.longlong))
     print(cython.sizeof(cython.uint) == cython.sizeof(cython.int))
     print(cython.sizeof(cython.p_int) == cython.sizeof(cython.p_double))
     if cython.compiled:
@@ -32,10 +36,6 @@
     (100, 100)
     >>> test_declare(100.5)
     (100, 100)
-    >>> test_declare(None) #doctest: +ELLIPSIS
-    Traceback (most recent call last):
-    ...
-    TypeError: ...
     """
     x = cython.declare(cython.int)
     y = cython.declare(cython.int, n)
@@ -103,7 +103,7 @@
 ##     return y
 
 
-def test_with_nogil(nogil):
+def test_with_nogil(nogil, should_raise=False):
     """
     >>> raised = []
     >>> class nogil(object):
@@ -118,27 +118,44 @@
     True
     >>> raised
     [None]
+
+    >>> test_with_nogil(nogil(), should_raise=True)
+    Traceback (most recent call last):
+    ValueError: RAISED!
+
+    >>> raised[1] is None
+    False
     """
     result = False
+    should_raise_bool = True if should_raise else False  # help the type inference ...
     with nogil:
         print("WORKS")
         with cython.nogil:
             result = True
+            if should_raise_bool:
+                raise ValueError("RAISED!")
     return result
 
+
 MyUnion = cython.union(n=cython.int, x=cython.double)
 MyStruct = cython.struct(is_integral=cython.bint, data=MyUnion)
 MyStruct2 = cython.typedef(MyStruct[2])
+MyStruct3 = cython.typedef(MyStruct[3])
 
 def test_struct(n, x):
     """
     >>> test_struct(389, 1.64493)
-    (389, 1.64493)
+    (389, 1.64493, False)
     """
-    a = cython.declare(MyStruct2)
+    a = cython.declare(MyStruct3)
     a[0] = MyStruct(is_integral=True, data=MyUnion(n=n))
     a[1] = MyStruct(is_integral=False, data={'x': x})
-    return a[0].data.n, a[1].data.x
+    if sys.version_info >= (3, 6):
+        # dict is ordered => struct creation via keyword arguments above was deterministic!
+        a[2] = MyStruct(False, MyUnion(x=x))
+    else:
+        a[2] = MyStruct(is_integral=False, data=MyUnion(x=x))
+    return a[0].data.n, a[1].data.x, a[2].is_integral
 
 import cython as cy
 from cython import declare, cast, locals, address, typedef, p_void, compiled
@@ -206,6 +223,12 @@
 @cython.ccall
 @cython.returns(cython.double)
 def c_call(x):
+    if x == -2.0:
+        raise RuntimeError("huhu!")
+    return x
+
+
+def call_ccall(x):
     """
     Test that a declared return type is honoured when compiled.
 
@@ -220,11 +243,11 @@
     1.0
     >>> (is_compiled and 1) or result
     1
-    """
-    return x
-
 
-def call_ccall(x):
+    >>> call_ccall(-2)
+    Traceback (most recent call last):
+    RuntimeError: huhu!
+    """
     ret = c_call(x)
     return ret, cython.typeof(ret)
 
@@ -233,9 +256,15 @@
 @cython.inline
 @cython.returns(cython.double)
 def cdef_inline(x):
+    if x == -2.0:
+        raise RuntimeError("huhu!")
+    return x + 1
+
+
+def call_cdef_inline(x):
     """
     >>> result, return_type = call_cdef_inline(1)
-    >>> (not is_compiled and 'float') or type(return_type).__name__
+    >>> (not is_compiled and 'float') or type(result).__name__
     'float'
     >>> (not is_compiled and 'double') or return_type
     'double'
@@ -243,13 +272,70 @@
     'int'
     >>> result == 2.0  or  result
     True
+
+    >>> call_cdef_inline(-2)
+    Traceback (most recent call last):
+    RuntimeError: huhu!
     """
+    ret = cdef_inline(x)
+    return ret, cython.typeof(ret)
+
+
+@cython.cfunc
+@cython.nogil
+@cython.locals(x=cython.int)
+@cython.returns(cython.int)
+def cdef_nogil(x):
     return x + 1
 
 
-def call_cdef_inline(x):
-    ret = cdef_inline(x)
-    return ret, cython.typeof(ret)
+@cython.cfunc
+@cython.nogil(True)
+@cython.locals(x=cython.int)
+@cython.returns(cython.int)
+def cdef_nogil_true(x):
+    return x + 1
+
+
+@cython.cfunc
+@cython.nogil(False)
+@cython.locals(x=cython.int)
+@cython.returns(cython.int)
+def cdef_nogil_false(x):
+    return x + 1
+
+
+@cython.locals(x=cython.int, result=cython.int)
+def test_cdef_nogil(x):
+    """
+    >>> test_cdef_nogil(5)
+    18
+    """
+    with cython.nogil:
+        result = cdef_nogil(x)
+    with cython.nogil(True):
+        result += cdef_nogil_true(x)
+    result += cdef_nogil_false(x)
+    return result
+
+
+@cython.cfunc
+@cython.inline
+def has_inner_func(x):
+    # the inner function must remain a Python function
+    # (and inline must not be applied to it)
+    @cython.test_fail_if_path_exists("//CFuncDefNode")
+    def inner():
+        return x
+    return inner
+
+
+def test_has_inner_func():
+    """
+    >>> test_has_inner_func()
+    1
+    """
+    return has_inner_func(1)()
 
 
 @cython.locals(counts=cython.int[10], digit=cython.int)
@@ -266,3 +352,252 @@
         assert 0 <= digit <= 9
         counts[digit] += 1
     return counts
+
+
+@cython.test_assert_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']")
+@cython.ccall
+@cython.returns(cython.long)
+@cython.exceptval(-1)
+def ccall_except(x):
+    """
+    >>> ccall_except(41)
+    42
+    >>> ccall_except(0)
+    Traceback (most recent call last):
+    ValueError
+    """
+    if x == 0:
+        raise ValueError
+    return x+1
+
+
+@cython.test_assert_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']")
+@cython.cfunc
+@cython.returns(cython.long)
+@cython.exceptval(-1)
+def cdef_except(x):
+    if x == 0:
+        raise ValueError
+    return x+1
+
+
+def call_cdef_except(x):
+    """
+    >>> call_cdef_except(41)
+    42
+    >>> call_cdef_except(0)
+    Traceback (most recent call last):
+    ValueError
+    """
+    return cdef_except(x)
+
+
+@cython.test_assert_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']")
+@cython.ccall
+@cython.returns(cython.long)
+@cython.exceptval(-1, check=True)
+def ccall_except_check(x):
+    """
+    >>> ccall_except_check(41)
+    42
+    >>> ccall_except_check(-2)
+    -1
+    >>> ccall_except_check(0)
+    Traceback (most recent call last):
+    ValueError
+    """
+    if x == 0:
+        raise ValueError
+    return x+1
+
+
+@cython.test_fail_if_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']")
+@cython.test_assert_path_exists("//CFuncDeclaratorNode")
+@cython.ccall
+@cython.returns(cython.long)
+@cython.exceptval(check=True)
+def ccall_except_check_always(x):
+    """
+    >>> ccall_except_check_always(41)
+    42
+    >>> ccall_except_check_always(0)
+    Traceback (most recent call last):
+    ValueError
+    """
+    if x == 0:
+        raise ValueError
+    return x+1
+
+
+@cython.test_fail_if_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']")
+@cython.test_assert_path_exists("//CFuncDeclaratorNode")
+@cython.ccall
+@cython.returns(cython.long)
+@cython.exceptval(check=False)
+def ccall_except_no_check(x):
+    """
+    >>> ccall_except_no_check(41)
+    42
+    >>> try: _ = ccall_except_no_check(0)  # no exception propagated!
+    ... except ValueError: assert not is_compiled
+    """
+    if x == 0:
+        raise ValueError
+    return x+1
+
+
+@cython.final
+@cython.cclass
+class CClass(object):
+    """
+    >>> c = CClass(2)
+    >>> c.get_attr()
+    int
+    2
+    """
+    cython.declare(attr=cython.int)
+
+    def __init__(self, attr):
+        self.attr = attr
+
+    def get_attr(self):
+        print(cython.typeof(self.attr))
+        return self.attr
+
+
+class TestUnboundMethod:
+    """
+    >>> C = TestUnboundMethod
+    >>> IS_PY2 or (C.meth is C.__dict__["meth"])
+    True
+    """
+    def meth(self): pass
+
+@cython.cclass
+class Foo:
+    a = cython.declare(cython.double)
+    b = cython.declare(cython.double)
+    c = cython.declare(cython.double)
+
+    @cython.locals(a=cython.double, b=cython.double, c=cython.double)
+    def __init__(self, a, b, c):
+        self.a = a
+        self.b = b
+        self.c = c
+
+@cython.cclass
+class EmptyClass(object):
+    def __init__(self, *args):
+        pass
+
+def same_type_cast():
+    """
+    >>> same_type_cast()
+    True
+    """
+
+    f = EmptyClass()
+    return f is cython.cast(EmptyClass, f)
+
+def multi_args_init_cast():
+    """
+    >>> multi_args_init_cast()
+    True
+    """
+    f = Foo(10, 20, 30)
+    return cython.cast(Foo, f) is f
+
+def multi_args_init_declare():
+    """
+    >>> multi_args_init_declare() is None
+    True
+    """
+    f = cython.declare(Foo)
+
+    if cython.compiled:
+        f = None
+
+    return f
+
+EmptyClassSyn = cython.typedef(EmptyClass)
+
+def empty_declare():
+    """
+    >>> empty_declare()
+    []
+    """
+
+    r0 = cython.declare(EmptyClass)
+    r1 = cython.declare(EmptyClassSyn)
+    r2 = cython.declare(MyStruct)
+    r3 = cython.declare(MyUnion)
+    r4 = cython.declare(MyStruct2)
+    r5 = cython.declare(cython.int[2])
+
+    if cython.compiled:
+        r0 = None
+        r1 = None
+
+    res = [
+        r0 is None,
+        r1 is None,
+        r2 is not None,
+        r3 is not None,
+        r4 is not None,
+        r5 is not None
+    ]
+
+    r2.is_integral = True
+    assert( r2.is_integral == True )
+
+    r3.x = 12.3
+    assert( r3.x == 12.3 )
+
+    #It generates a correct C code, but raises an exception when interpreted
+    if cython.compiled:
+        r4[0].is_integral = True
+        assert( r4[0].is_integral == True )
+
+    r5[0] = 42
+    assert ( r5[0] == 42 )
+
+    return [i for i, x in enumerate(res) if not x]
+
+def same_declare():
+    """
+    >>> same_declare()
+    True
+    """
+
+    f = EmptyClass()
+    f2 = cython.declare(EmptyClass, f)
+    return f2 is f
+
+def none_cast():
+    """
+    >>> none_cast() is None
+    True
+    """
+
+    f = None
+    return cython.cast(EmptyClass, f)
+
+def none_declare():
+    """
+    >>> none_declare() is None
+    True
+    """
+
+    f = None
+    f2 = cython.declare(Foo, f)
+    return f2
+
+def array_init_with_list():
+    """
+    >>> array_init_with_list()
+    [10, 42]
+    """
+    x = cython.declare(cython.int[20], list(range(20)))
+    x[12] = 42
+
+    return [x[10], x[12]]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure.pyx cython-0.20.1+1~202203241016-9537/tests/run/pure.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pure.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -25,10 +25,12 @@
     (100, 100)
     >>> test_declare(100.5)
     (100, 100)
-    >>> test_declare(None)
+
+    # CPython: "TypeError: an integer is required"
+    >>> test_declare(None) # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     x = cython.declare(cython.int)
     y = cython.declare(cython.int, n)
@@ -160,3 +162,23 @@
     #z01 = cython.declare(cython.floatcomplex, n+1j)
     #z02 = cython.declare(cython.doublecomplex, n+1j)
     #z03 = cython.declare(cython.longdoublecomplex, n+1j)
+
+
+cdef class ExtType:
+    """
+    >>> x = ExtType()
+    >>> x.forward_ref(x)
+    'ExtType'
+    """
+    @cython.locals(x="ExtType")
+    def forward_ref(self, x):
+        return cython.typeof(x)
+
+
+def ext_type_string_ref(x: "ExtType"):
+    """
+    >>> x = ExtType()
+    >>> ext_type_string_ref(x)
+    'ExtType'
+    """
+    return cython.typeof(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pure_pyx_cimports.pyx cython-0.20.1+1~202203241016-9537/tests/run/pure_pyx_cimports.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pure_pyx_cimports.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pure_pyx_cimports.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,19 @@
+# mode: run
+# tag: pure, import, cimport
+
+cimport cython.cimports.libc.math as libc_math1
+
+from cython.cimports.libc import math as libc_math2
+from cython.cimports.libc.math import ceil as math_ceil
+
+#from cython.cimports cimport libc    # FIXME: currently crashes during analysis when submodule cannot be found
+from cython.cimports.libc cimport math
+from cython.cimports.libc.math cimport ceil
+
+
+def libc_math_ceil(x):
+    """
+    >>> libc_math_ceil(1.5)
+    [2, 2, 2, 2, 2]
+    """
+    return [int(n) for n in [ceil(x), math.ceil(x), libc_math1.ceil(x), libc_math2.ceil(x), math_ceil(x)]]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pxd_argument_names.srctree cython-0.20.1+1~202203241016-9537/tests/run/pxd_argument_names.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/pxd_argument_names.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pxd_argument_names.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,52 @@
+# mode: run
+# ticket: 1888
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import a; a.test()"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup
+
+setup(
+    ext_modules=cythonize("a.pyx"),
+)
+
+######## a.pxd ########
+
+cdef int do_stuff(int foo) except -1
+
+######## a.pyx ########
+
+cdef int do_stuff(int bar) except -1:
+    if bar == 0:
+        raise ValueError()
+    return bar
+
+
+cdef call_do_stuff(int x):
+    # The keyword argument name is surprising, but actually correct.
+    # The caller signature is defined by the .pxd file, not the implementation.
+    return do_stuff(foo=x)
+
+
+def test():
+    assert do_stuff(1) == 1
+    assert do_stuff(foo=1) == 1
+    assert call_do_stuff(1) == 1
+
+    try:
+        do_stuff(0)
+    except ValueError:
+        pass
+    else:
+        assert False, "exception not raised"
+
+    try:
+        call_do_stuff(0)
+    except ValueError:
+        pass
+    else:
+        assert False, "exception not raised"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pxd_signature_excvalue.srctree cython-0.20.1+1~202203241016-9537/tests/run/pxd_signature_excvalue.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/pxd_signature_excvalue.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pxd_signature_excvalue.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,59 @@
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import foo"
+PYTHON -c "import a; a.test()"
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("*.pyx"),
+)
+
+######## foo.pxd ########
+
+cdef int bar(int i) except *
+
+cdef int (*var_opt)(int) except? -1
+cdef int (*var_orig)(int) except *
+
+######## foo.pyx ########
+
+cdef int bar(int i) except *:
+    if i == 10:
+        raise ValueError()
+    return i + 1
+
+var_opt = bar   # by 'accident' of optimisation
+var_orig = bar  # by declaration
+
+######## a.pyx ########
+
+cimport cython
+from foo cimport bar, var_orig, var_opt
+
+def test():
+    assert bar(-2) == -1
+    try:
+        bar(10)
+    except ValueError:
+        pass
+    else:
+        assert False, "exception not raised in bar()"
+
+    assert var_orig(-2) == -1
+    try:
+        var_orig(10)
+    except ValueError:
+        pass
+    else:
+        assert False, "exception not raised in var_orig()"
+
+    assert var_opt(-2) == -1
+    try:
+        var_opt(10)
+    except ValueError:
+        pass
+    else:
+        assert False, "exception not raised in var_opt()"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py34_signature.pyx cython-0.20.1+1~202203241016-9537/tests/run/py34_signature.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py34_signature.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py34_signature.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -4,7 +4,10 @@
 
 import inspect
 
-sig = inspect.Signature.from_function
+try:
+    sig = inspect.Signature.from_callable
+except AttributeError:
+    sig = inspect.Signature.from_function
 
 
 def signatures_match(f1, f2):
@@ -89,3 +92,26 @@
     >>> def py_n(a, *, b, c = 88): pass
     >>> signatures_match(n, py_n)
     """
+
+
+cpdef cp1(a, b):
+    """
+    >>> def py_cp1(a, b): pass
+    >>> signatures_match(cp1, py_cp1)
+    """
+
+
+cpdef cp2(a, b=True):
+    """
+    >>> def py_cp2(a, b=True): pass
+
+    >>> signatures_match(cp2, py_cp2)
+    """
+
+
+cpdef cp3(a=1, b=True):
+    """
+    >>> def py_cp3(a=1, b=True): pass
+
+    >>> signatures_match(cp3, py_cp3)
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py35_asyncio_async_def.srctree cython-0.20.1+1~202203241016-9537/tests/run/py35_asyncio_async_def.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/py35_asyncio_async_def.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py35_asyncio_async_def.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,76 @@
+# mode: run
+# tag: asyncio, gh1685, gh2273
+
+PYTHON setup.py build_ext -i
+PYTHON main.py
+
+
+######## setup.py ########
+
+from Cython.Build import cythonize
+from distutils.core import setup
+
+setup(
+    ext_modules = cythonize("*.pyx"),
+)
+
+
+######## main.py ########
+
+import asyncio
+import cy_test
+import py_test
+from contextlib import closing
+
+async def main():
+    await cy_test.say()
+
+with closing(asyncio.get_event_loop()) as loop:
+    print("Running Python coroutine ...")
+    loop.run_until_complete(main())
+
+    print("Running Cython coroutine ...")
+    loop.run_until_complete(cy_test.say())
+
+assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True
+assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True
+assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True
+assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True
+assert asyncio.iscoroutinefunction(cy_test.cy_def_example) == False
+assert asyncio.iscoroutinefunction(py_test.py_def_example) == False
+
+######## cy_test.pyx ########
+
+import asyncio
+from py_test import py_async
+
+async def cy_async():
+    print("- this one is from Cython")
+
+async def say():
+    await cb()
+
+async def cb():
+    print("awaiting:")
+    await cy_async()
+    await py_async()
+    print("sleeping:")
+    await asyncio.sleep(0.5)
+    print("done!")
+
+async def cy_async_def_example():
+    return 1
+
+def cy_def_example():
+    return 1
+
+######## py_test.py ########
+
+async def py_async():
+    print("- and this one is from Python")
+
+async def py_async_def_example():
+    return 1
+
+def py_def_example():
+    return 1
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py35_pep492_interop.pyx cython-0.20.1+1~202203241016-9537/tests/run/py35_pep492_interop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py35_pep492_interop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py35_pep492_interop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,9 +3,10 @@
 # tag: pep492, asyncfor, await
 
 
-def run_async(coro):
-    #assert coro.__class__ is types.GeneratorType
-    assert coro.__class__.__name__ in ('coroutine', 'GeneratorWrapper'), coro.__class__.__name__
+def run_async(coro, ignore_type=False):
+    if not ignore_type:
+        #assert coro.__class__ is types.GeneratorType
+        assert coro.__class__.__name__ in ('coroutine', 'GeneratorWrapper'), coro.__class__.__name__
 
     buffer = []
     result = None
@@ -87,3 +88,69 @@
         return await awaitable
 
     return simple, awaiting
+
+
+cimport cython
+
+def yield_from_cyobject():
+    """
+    >>> async def py_simple_nonit():
+    ...     return 10
+
+    >>> async def run_await(awaitable):
+    ...     return await awaitable
+
+    >>> def run_yield_from(it):
+    ...     return (yield from it)
+
+    >>> simple_nonit, simple_it, awaiting, yield_from = yield_from_cyobject()
+
+    >>> buffer, result = run_async(run_await(simple_it()))
+    >>> result
+    10
+    >>> buffer, result = run_async(run_await(awaiting(simple_it())))
+    >>> result
+    10
+    >>> buffer, result = run_async(awaiting(run_await(simple_it())), ignore_type=True)
+    >>> result
+    10
+    >>> buffer, result = run_async(run_await(py_simple_nonit()))
+    >>> result
+    10
+
+    >>> buffer, result = run_async(run_yield_from(awaiting(run_await(simple_it()))), ignore_type=True)
+    >>> result
+    10
+
+    >>> buffer, result = run_async(run_yield_from(simple_it()), ignore_type=True)
+    >>> result
+    10
+    >>> buffer, result = run_async(yield_from(simple_it()), ignore_type=True)
+    >>> result
+    10
+
+    >>> next(run_yield_from(simple_nonit()))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    >>> next(run_yield_from(py_simple_nonit()))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...
+    >>> next(yield_from(py_simple_nonit()))
+    Traceback (most recent call last):
+    TypeError: 'coroutine' object is not iterable
+    """
+    async def simple_nonit():
+        return 10
+
+    @cython.iterable_coroutine
+    async def simple_it():
+        return 10
+
+    @cython.iterable_coroutine
+    async def awaiting(awaitable):
+        return await awaitable
+
+    def yield_from(it):
+        return (yield from it)
+
+    return simple_nonit, simple_it, awaiting, yield_from
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py3k_super.pyx cython-0.20.1+1~202203241016-9537/tests/run/py3k_super.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py3k_super.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py3k_super.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: py3k_super
+# tag: py3k_super, gh3246
 
 class A(object):
     def method(self):
@@ -89,3 +89,21 @@
 #         return super().method_cp()
 #     cdef method_c(self):
 #         return super().method_c()
+
+
+def freeing_class_cell_temp_gh3246():
+    # https://github.com/cython/cython/issues/3246
+    """
+    >>> abc = freeing_class_cell_temp_gh3246()
+    >>> abc().a
+    1
+    """
+    class SimpleBase(object):
+        def __init__(self):
+            self.a = 1
+
+    class ABC(SimpleBase):
+        def __init__(self):
+            super().__init__()
+
+    return ABC
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyclass_annotations_pep526.py cython-0.20.1+1~202203241016-9537/tests/run/pyclass_annotations_pep526.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pyclass_annotations_pep526.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyclass_annotations_pep526.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,59 @@
+# cython: language_level=3
+# mode: run
+# tag: pure3.7, pep526, pep484
+
+from __future__ import annotations
+
+import sys
+
+try:
+    from typing import ClassVar
+except ImportError:  # Py<=3.5
+    ClassVar = {int: int}
+
+class NotAStr:
+    pass
+
+class PyAnnotatedClass:
+    """
+    >>> PyAnnotatedClass.__annotations__["CLASS_VAR"]
+    'ClassVar[int]'
+    >>> PyAnnotatedClass.__annotations__["obj"]
+    'str'
+    >>> PyAnnotatedClass.__annotations__["literal"]
+    "'int'"
+    >>> PyAnnotatedClass.__annotations__["recurse"]
+    "'PyAnnotatedClass'"
+    >>> PyAnnotatedClass.__annotations__["default"]
+    'bool'
+    >>> PyAnnotatedClass.CLASS_VAR
+    1
+    >>> PyAnnotatedClass.default
+    False
+    >>> PyAnnotatedClass.obj
+    Traceback (most recent call last):
+      ...
+    AttributeError: type object 'PyAnnotatedClass' has no attribute 'obj'
+    """
+    CLASS_VAR: ClassVar[int] = 1
+    obj: str
+    literal: "int"
+    recurse: "PyAnnotatedClass"
+    default: bool = False
+    # https://github.com/cython/cython/issues/4196 and https://github.com/cython/cython/issues/4198
+    not_object: float = 0.1  # Shouldn't try to create a c attribute
+    lie_about_type: str = NotAStr  # Shouldn't generate a runtime type-check
+
+
+class PyVanillaClass:
+    """
+    Before Py3.10, unannotated classes did not have '__annotations__'.
+
+    >>> try:
+    ...     a = PyVanillaClass.__annotations__
+    ... except AttributeError:
+    ...     assert sys.version_info < (3, 10)
+    ... else:
+    ...     assert sys.version_info >= (3, 10)
+    ...     assert a == {}
+    """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyclass_dynamic_bases.pyx cython-0.20.1+1~202203241016-9537/tests/run/pyclass_dynamic_bases.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pyclass_dynamic_bases.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyclass_dynamic_bases.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -24,3 +24,22 @@
     class PyClass(A if x else B):
         p = 5
     return PyClass
+
+
+def make_subclass(*bases):
+    """
+    >>> cls = make_subclass(list)
+    >>> issubclass(cls, list) or cls.__mro__
+    True
+
+    >>> class Cls(object): pass
+    >>> cls = make_subclass(Cls, list)
+    >>> issubclass(cls, list) or cls.__mro__
+    True
+    >>> issubclass(cls, Cls) or cls.__mro__
+    True
+    """
+    # GH-3338
+    class MadeClass(*bases):
+        pass
+    return MadeClass
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyclass_scope_T671.py cython-0.20.1+1~202203241016-9537/tests/run/pyclass_scope_T671.py
--- cython-0.20.1+1~201611251650-6686/tests/run/pyclass_scope_T671.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyclass_scope_T671.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 671
+# ticket: t671
 
 A = 1234
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pycontextvar.pyx cython-0.20.1+1~202203241016-9537/tests/run/pycontextvar.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pycontextvar.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pycontextvar.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,56 @@
+# mode: run
+
+from cpython.contextvars cimport (
+    PyContextVar_New, PyContextVar_New_with_default,
+    get_value, get_value_no_default,
+)
+
+NOTHING = object()
+CVAR = PyContextVar_New("cvar", NULL)
+CVAR_WITH_DEFAULT = PyContextVar_New_with_default("cvar_wd", "DEFAULT")
+
+
+import contextvars
+PYCVAR = contextvars.ContextVar("pycvar")
+
+def disable_for_pypy737(f):
+    import sys
+    # will be fixed in PyPy 7.3.8
+    if hasattr(sys, 'pypy_version_info') and sys.pypy_version_info < (7,3,8):
+        return None
+    return f
+
+
+@disable_for_pypy737
+def test_get_value(var, default=NOTHING):
+    """
+    >>> test_get_value(CVAR)
+    >>> test_get_value(CVAR, "default")
+    'default'
+    >>> test_get_value(PYCVAR)
+    >>> test_get_value(PYCVAR, "default")
+    'default'
+    >>> test_get_value(CVAR_WITH_DEFAULT)
+    'DEFAULT'
+    >>> test_get_value(CVAR_WITH_DEFAULT, "default")
+    'DEFAULT'
+    """
+    return get_value(var, default) if default is not NOTHING else get_value(var)
+
+
+@disable_for_pypy737
+def test_get_value_no_default(var, default=NOTHING):
+    """
+    >>> test_get_value_no_default(CVAR)
+    >>> test_get_value_no_default(CVAR, "default")
+    'default'
+    >>> test_get_value_no_default(PYCVAR)
+    >>> test_get_value_no_default(PYCVAR, "default")
+    'default'
+    >>> test_get_value_no_default(CVAR_WITH_DEFAULT)
+    >>> test_get_value_no_default(CVAR_WITH_DEFAULT, "default")
+    'default'
+    """
+    return get_value_no_default(var, default) if default is not NOTHING else get_value_no_default(var)
+
+__test__ = {}
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyfunction_redefine_T489.pyx cython-0.20.1+1~202203241016-9537/tests/run/pyfunction_redefine_T489.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pyfunction_redefine_T489.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyfunction_redefine_T489.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 489
+# ticket: t489
 
 """
 >>> xxx
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py_hash_t.pyx cython-0.20.1+1~202203241016-9537/tests/run/py_hash_t.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py_hash_t.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py_hash_t.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,12 +2,30 @@
 cimport cython
 
 
+class IntLike(object):
+  def __init__(self, value):
+    self.value = value
+  def __index__(self):
+    return self.value
+
+
 def assign_py_hash_t(x):
     """
     >>> assign_py_hash_t(12)
     12
     >>> assign_py_hash_t(-12)
     -12
+
+    >>> assign_py_hash_t(IntLike(-3))
+    -3
+    >>> assign_py_hash_t(IntLike(1 << 100))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    OverflowError: ...
+    >>> assign_py_hash_t(IntLike(1.5))  # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    ...
+    TypeError: __index__ ... (type ...float...)
     """
     cdef Py_hash_t h = x
     return h
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyintop.pyx cython-0.20.1+1~202203241016-9537/tests/run/pyintop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pyintop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyintop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,7 @@
 # mode: run
 
+cimport cython
+
 
 def bigint(x):
     # avoid 'L' postfix in Py2.x
@@ -10,6 +12,7 @@
     print(str(x).replace('L', ''))
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
 def or_obj(obj2, obj3):
     """
     >>> or_obj(2, 3)
@@ -19,8 +22,11 @@
     return obj1
 
 
+@cython.test_fail_if_path_exists('//IntBinopNode')
 def or_int(obj2):
     """
+    >>> or_int(0)
+    16
     >>> or_int(1)
     17
     >>> or_int(16)
@@ -30,6 +36,7 @@
     return obj1
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
 def xor_obj(obj2, obj3):
     """
     >>> xor_obj(2, 3)
@@ -39,8 +46,11 @@
     return obj1
 
 
+@cython.test_fail_if_path_exists('//IntBinopNode')
 def xor_int(obj2):
     """
+    >>> xor_int(0)
+    16
     >>> xor_int(2)
     18
     >>> xor_int(16)
@@ -50,6 +60,7 @@
     return obj1
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
 def and_obj(obj2, obj3):
     """
     >>> and_obj(2, 3)
@@ -59,17 +70,23 @@
     return obj1
 
 
+@cython.test_fail_if_path_exists('//IntBinopNode')
 def and_int(obj2):
     """
+    >>> and_int(0)
+    0
     >>> and_int(1)
     0
     >>> and_int(18)
     16
+    >>> and_int(-1)
+    16
     """
     obj1 = obj2 & 0x10
     return obj1
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
 def lshift_obj(obj2, obj3):
     """
     >>> lshift_obj(2, 3)
@@ -79,6 +96,7 @@
     return obj1
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
 def rshift_obj(obj2, obj3):
     """
     >>> rshift_obj(2, 3)
@@ -88,8 +106,30 @@
     return obj1
 
 
+@cython.test_assert_path_exists('//IntBinopNode')
+def rshift_int_obj(obj3):
+    """
+    >>> rshift_int_obj(3)
+    0
+    >>> rshift_int_obj(2)
+    0
+    >>> rshift_int_obj(1)
+    1
+    >>> rshift_int_obj(0)
+    2
+    >>> rshift_int_obj(-1)
+    Traceback (most recent call last):
+    ValueError: negative shift count
+    """
+    obj1 = 2 >> obj3
+    return obj1
+
+
+@cython.test_fail_if_path_exists('//IntBinopNode')
 def rshift_int(obj2):
     """
+    >>> rshift_int(0)
+    0
     >>> rshift_int(2)
     0
 
@@ -134,6 +174,10 @@
     return obj1
 
 
+@cython.test_assert_path_exists(
+    '//SingleAssignmentNode//IntBinopNode',
+    '//SingleAssignmentNode//PythonCapiCallNode',
+)
 def lshift_int(obj):
     """
     >>> lshift_int(0)
@@ -159,16 +203,16 @@
     >>> bigints(lshift_int(-32))
     (-256, -68719476736, -295147905179352825856, -340282366920938463463374607431768211456)
 
-    >>> (2**28) << 3
+    >>> bigint((2**28) << 3)
     2147483648
     >>> bigints(lshift_int(2**28))
     (2147483648, 576460752303423488, 2475880078570760549798248448, 2854495385411919762116571938898990272765493248)
-    >>> (-2**28) << 3
+    >>> bigint((-2**28) << 3)
     -2147483648
     >>> bigints(lshift_int(-2**28))
     (-2147483648, -576460752303423488, -2475880078570760549798248448, -2854495385411919762116571938898990272765493248)
 
-    >>> (2**30) << 3
+    >>> bigint((2**30) << 3)
     8589934592
     >>> bigints(lshift_int(2**30))
     (8589934592, 2305843009213693952, 9903520314283042199192993792, 11417981541647679048466287755595961091061972992)
@@ -189,6 +233,10 @@
     return r1, r2, r3, r4
 
 
+@cython.test_assert_path_exists(
+    '//IntBinopNode',
+    '//IntBinopNode//IntBinopNode',
+)
 def mixed_obj(obj2, obj3):
     """
     >>> mixed_obj(2, 3)
@@ -198,8 +246,17 @@
     return obj1
 
 
+@cython.test_assert_path_exists(
+    '//IntBinopNode',
+    '//IntBinopNode//PythonCapiCallNode',
+)
+@cython.test_fail_if_path_exists(
+    '//IntBinopNode//IntBinopNode',
+)
 def mixed_int(obj2):
     """
+    >>> mixed_int(0)
+    16
     >>> mixed_int(2)
     18
     >>> mixed_int(16)
@@ -209,3 +266,244 @@
     """
     obj1 = (obj2 ^ 0x10) | (obj2 & 0x01)
     return obj1
+
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists(
+    '//IntBinopNode',
+    '//PrimaryCmpNode',
+)
+def equals(obj2):
+    """
+    >>> equals(2)
+    True
+    >>> equals(0)
+    False
+    >>> equals(-1)
+    False
+    """
+    result = obj2 == 2
+    return result
+
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists(
+    '//IntBinopNode',
+    '//PrimaryCmpNode',
+)
+def not_equals(obj2):
+    """
+    >>> not_equals(2)
+    False
+    >>> not_equals(0)
+    True
+    >>> not_equals(-1)
+    True
+    """
+    result = obj2 != 2
+    return result
+
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_assert_path_exists('//PrimaryCmpNode')
+def equals_many(obj2):
+    """
+    >>> equals_many(-2)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(0)
+    (True, False, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(1)
+    (False, True, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(-1)
+    (False, False, True, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(2**30)
+    (False, False, False, True, False, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(-2**30)
+    (False, False, False, False, True, False, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(2**30-1)
+    (False, False, False, False, False, True, False, False, False, False, False, False, False, False, False)
+    >>> equals_many(-2**30+1)
+    (False, False, False, False, False, False, True, False, False, False, False, False, False, False, False)
+    >>> equals_many(2**32)
+    (False, False, False, False, False, False, False, True, False, False, False, False, False, False, False)
+    >>> equals_many(-2**32)
+    (False, False, False, False, False, False, False, False, True, False, False, False, False, False, False)
+    >>> equals_many(2**45-1)
+    (False, False, False, False, False, False, False, False, False, True, False, False, False, False, False)
+    >>> equals_many(-2**45+1)
+    (False, False, False, False, False, False, False, False, False, False, True, False, False, False, False)
+    >>> equals_many(2**64)
+    (False, False, False, False, False, False, False, False, False, False, False, True, False, False, False)
+    >>> equals_many(-2**64)
+    (False, False, False, False, False, False, False, False, False, False, False, False, True, False, False)
+    >>> equals_many(2**64-1)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, True, False)
+    >>> equals_many(-2**64+1)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, False, True)
+    """
+    cdef bint x, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o
+    a = obj2 == 0
+    x = 0 == obj2
+    assert a == x
+    b = obj2 == 1
+    x = 1 == obj2
+    assert b == x
+    c = obj2 == -1
+    x = -1 == obj2
+    assert c == x
+    d = obj2 == 2**30
+    x = 2**30 == obj2
+    assert d == x
+    e = obj2 == -2**30
+    x = -2**30 == obj2
+    assert e == x
+    f = obj2 == 2**30-1
+    x = 2**30-1 == obj2
+    assert f == x
+    g = obj2 == -2**30+1
+    x = -2**30+1 == obj2
+    assert g == x
+    h = obj2 == 2**32
+    x = 2**32 == obj2
+    assert h == x
+    i = obj2 == -2**32
+    x = -2**32 == obj2
+    assert i == x
+    j = obj2 == 2**45-1
+    x = 2**45-1 == obj2
+    assert j == x
+    k = obj2 == -2**45+1
+    x = -2**45+1 == obj2
+    assert k == x
+    l = obj2 == 2**64
+    x = 2**64 == obj2
+    assert l == x
+    m = obj2 == -2**64
+    x = -2**64 == obj2
+    assert m == x
+    n = obj2 == 2**64-1
+    x = 2**64-1 == obj2
+    assert n == x
+    o = obj2 == -2**64+1
+    x = -2**64+1 == obj2
+    assert o == x
+    return (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o)
+
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_assert_path_exists('//PrimaryCmpNode')
+def not_equals_many(obj2):
+    """
+    >>> not_equals_many(-2)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(0)
+    (True, False, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(1)
+    (False, True, False, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(-1)
+    (False, False, True, False, False, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(2**30)
+    (False, False, False, True, False, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(-2**30)
+    (False, False, False, False, True, False, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(2**30-1)
+    (False, False, False, False, False, True, False, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(-2**30+1)
+    (False, False, False, False, False, False, True, False, False, False, False, False, False, False, False)
+    >>> not_equals_many(2**32)
+    (False, False, False, False, False, False, False, True, False, False, False, False, False, False, False)
+    >>> not_equals_many(-2**32)
+    (False, False, False, False, False, False, False, False, True, False, False, False, False, False, False)
+    >>> not_equals_many(2**45-1)
+    (False, False, False, False, False, False, False, False, False, True, False, False, False, False, False)
+    >>> not_equals_many(-2**45+1)
+    (False, False, False, False, False, False, False, False, False, False, True, False, False, False, False)
+    >>> not_equals_many(2**64)
+    (False, False, False, False, False, False, False, False, False, False, False, True, False, False, False)
+    >>> not_equals_many(-2**64)
+    (False, False, False, False, False, False, False, False, False, False, False, False, True, False, False)
+    >>> not_equals_many(2**64-1)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, True, False)
+    >>> not_equals_many(-2**64+1)
+    (False, False, False, False, False, False, False, False, False, False, False, False, False, False, True)
+    """
+    cdef bint a, b, c, d, e, f, g, h, i, j, k, l, m, n, o
+    a = obj2 != 0
+    x = 0 != obj2
+    assert a == x
+    b = obj2 != 1
+    x = 1 != obj2
+    assert b == x
+    c = obj2 != -1
+    x = -1 != obj2
+    assert c == x
+    d = obj2 != 2**30
+    x = 2**30 != obj2
+    assert d == x
+    e = obj2 != -2**30
+    x = -2**30 != obj2
+    assert e == x
+    f = obj2 != 2**30-1
+    x = 2**30-1 != obj2
+    assert f == x
+    g = obj2 != -2**30+1
+    x = -2**30+1 != obj2
+    assert g == x
+    h = obj2 != 2**32
+    x = 2**32 != obj2
+    assert h == x
+    i = obj2 != -2**32
+    x = -2**32 != obj2
+    assert i == x
+    j = obj2 != 2**45-1
+    x = 2**45-1 != obj2
+    assert j == x
+    k = obj2 != -2**45+1
+    x = -2**45+1 != obj2
+    assert k == x
+    l = obj2 != 2**64
+    x = 2**64 != obj2
+    assert l == x
+    m = obj2 != -2**64
+    x = -2**64 != obj2
+    assert m == x
+    n = obj2 != 2**64-1
+    x = 2**64-1 != obj2
+    assert n == x
+    o = obj2 != -2**64+1
+    x = -2**64+1 != obj2
+    assert o == x
+    return tuple(not x for x in (a, b, c, d, e, f, g, h, i, j, k, l, m, n, o))
+
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists(
+    '//IntBinopNode',
+    '//PrimaryCmpNode',
+)
+def equals_zero(obj2):
+    """
+    >>> equals_zero(2)
+    False
+    >>> equals_zero(0)
+    True
+    >>> equals_zero(-1)
+    False
+    """
+    result = obj2 == 0
+    return result
+
+
+def truthy(obj2):
+    """
+    >>> truthy(2)
+    True
+    >>> truthy(0)
+    False
+    >>> truthy(-1)
+    True
+    """
+    if obj2:
+        return True
+    else:
+        return False
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/pyobjcast_T313.pyx cython-0.20.1+1~202203241016-9537/tests/run/pyobjcast_T313.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/pyobjcast_T313.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/pyobjcast_T313.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 313
+# ticket: t313
 # Ensure casting still works to void*
 
 """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/python_bool_type.pyx cython-0.20.1+1~202203241016-9537/tests/run/python_bool_type.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/python_bool_type.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/python_bool_type.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -132,18 +132,18 @@
     assertIs(0!=1, True)
     assertIs(0!=0, False)
 
-    x = [1]
-    assertIs(x is x, True)
-    assertIs(x is not x, False)
+    y = x = [1]
+    assertIs(x is y, True)
+    assertIs(x is not y, False)
 
     assertIs(1 in x, True)
     assertIs(0 in x, False)
     assertIs(1 not in x, False)
     assertIs(0 not in x, True)
 
-    x = {1: 2}
-    assertIs(x is x, True)
-    assertIs(x is not x, False)
+    y = x = {1: 2}
+    assertIs(x is y, True)
+    assertIs(x is not y, False)
 
     assertIs(1 in x, True)
     assertIs(0 in x, False)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py_ucs4_type.pyx cython-0.20.1+1~202203241016-9537/tests/run/py_ucs4_type.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py_ucs4_type.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py_ucs4_type.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,7 @@
 # -*- coding: iso-8859-1 -*-
+# mode: run
+# tag: warnings
+
 
 cimport cython
 
@@ -129,15 +132,24 @@
         uchar.isupper(),
         ]
 
-@cython.test_assert_path_exists('//PythonCapiCallNode')
-@cython.test_fail_if_path_exists('//SimpleCallNode')
+#@cython.test_assert_path_exists('//PythonCapiCallNode')
+#@cython.test_fail_if_path_exists('//SimpleCallNode')
 def unicode_methods(Py_UCS4 uchar):
     """
-    >>> unicode_methods(ord('A')) == ['a', 'A', 'A']
+    >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] or unicode_methods(ord('A'))
+    True
+    >>> unicode_methods(ord('a')) == ['a', 'A', 'A'] or unicode_methods(ord('a'))
+    True
+    >>> unicode_methods(0x1E9E) == [u'\\xdf', u'\\u1e9e', u'\\u1e9e'] or unicode_methods(0x1E9E)
     True
-    >>> unicode_methods(ord('a')) == ['a', 'A', 'A']
+    >>> unicode_methods(0x0130) in (
+    ...     [u'i\\u0307', u'\\u0130', u'\\u0130'],  # Py3
+    ...     [u'i', u'\\u0130', u'\\u0130'],  # Py2
+    ... ) or unicode_methods(0x0130)
     True
     """
+    # \u1E9E == 'LATIN CAPITAL LETTER SHARP S'
+    # \u0130 == 'LATIN CAPITAL LETTER I WITH DOT ABOVE'
     return [
         # character conversion
         uchar.lower(),
@@ -146,11 +158,11 @@
         ]
 
 
-@cython.test_assert_path_exists('//PythonCapiCallNode')
-@cython.test_fail_if_path_exists(
-    '//SimpleCallNode',
-    '//CoerceFromPyTypeNode',
-)
+#@cython.test_assert_path_exists('//PythonCapiCallNode')
+#@cython.test_fail_if_path_exists(
+#    '//SimpleCallNode',
+#    '//CoerceFromPyTypeNode',
+#)
 def unicode_method_return_type(Py_UCS4 uchar):
     """
     >>> unicode_method_return_type(ord('A'))
@@ -342,3 +354,26 @@
     """
     assert uchar == 0x12345, ('%X' % uchar)
     return uchar in ustring
+
+
+def uchar_lookup_in_dict(obj, Py_UCS4 uchar):
+    """
+    >>> d = {u_KLINGON: 1234, u0: 0, u1: 1, u_A: 2}
+    >>> uchar_lookup_in_dict(d, u_KLINGON)
+    (1234, 1234)
+    >>> uchar_lookup_in_dict(d, u_A)
+    (2, 2)
+    >>> uchar_lookup_in_dict(d, u0)
+    (0, 0)
+    >>> uchar_lookup_in_dict(d, u1)
+    (1, 1)
+    """
+    cdef dict d = obj
+    dval = d[uchar]
+    objval = obj[uchar]
+    return dval, objval
+
+
+_WARNINGS = """
+373:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour.
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py_unicode_strings.pyx cython-0.20.1+1~202203241016-9537/tests/run/py_unicode_strings.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py_unicode_strings.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py_unicode_strings.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,15 +1,15 @@
+# mode: run
 # tag: py_unicode_strings
 
 import sys
 
-cimport cython
-from libc.string cimport memcpy, strcpy
+from libc.string cimport memcpy
 
-cdef bint Py_UNICODE_equal(const Py_UNICODE* u1, const Py_UNICODE* u2):
-    while u1[0] != 0 and u2[0] != 0 and u1[0] == u2[0]:
-        u1 += 1
-        u2 += 1
-    return u1[0] == u2[0]
+cdef assert_Py_UNICODE_equal(const Py_UNICODE* u1, const Py_UNICODE* u2):
+    cdef size_t i = 0
+    while u1[i] != 0 and u2[i] != 0 and u1[i] == u2[i]:
+        i += 1
+    assert u1[i] == u2[i], f"Mismatch at position {i}: {u1[i]} != {u2[i]} ({u1!r} != {u2!r})"
 
 
 ctypedef Py_UNICODE* LPWSTR
@@ -48,9 +48,10 @@
     assert c_pu_str[1:7] == uobj[1:7]
     assert c_wstr[1:7] == uobj[1:7]
 
-    assert c_pu_arr[1] == uobj[1]
-    assert c_pu_str[1] == uobj[1]
-    assert c_wstr[1] == uobj[1]
+    cdef Py_UNICODE ch = uobj[1]  # Py_UCS4 is unsigned, Py_UNICODE is usually signed.
+    assert c_pu_arr[1] == ch
+    assert c_pu_str[1] == ch
+    assert c_wstr[1] == ch
 
     assert len(c_pu_str) == 8
     assert len(c_pu_arr) == 8
@@ -81,20 +82,20 @@
     """
     cdef unicode u
 
-    assert Py_UNICODE_equal(c_pu_arr, uobj)
-    assert Py_UNICODE_equal(c_pu_str, uobj)
-    assert Py_UNICODE_equal(c_pu_str, uobj)
+    assert_Py_UNICODE_equal(c_pu_arr, uobj)
+    assert_Py_UNICODE_equal(c_pu_str, uobj)
+    assert_Py_UNICODE_equal(c_pu_str, uobj)
     u = uobj[1:]
-    assert Py_UNICODE_equal(c_pu_str + 1, u)
-    assert Py_UNICODE_equal(c_wstr + 1, u)
+    assert_Py_UNICODE_equal(c_pu_str + 1, u)
+    assert_Py_UNICODE_equal(c_wstr + 1, u)
     u = uobj[:1]
-    assert Py_UNICODE_equal(u"u", u)
+    assert_Py_UNICODE_equal(u"u", u)
     u = uobj[1:7]
-    assert Py_UNICODE_equal(u"nicode", u)
+    assert_Py_UNICODE_equal(u"nicode", u)
     u = uobj[1]
-    assert Py_UNICODE_equal(u"n", u)
+    assert_Py_UNICODE_equal(u"n", u)
 
-    assert Py_UNICODE_equal(uwide_literal, c_pu_wide_literal)
+    assert_Py_UNICODE_equal(uwide_literal, c_pu_wide_literal)
 
     assert len(u"abc\0") == 4
     assert len(u"abc\0") == 3
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/py_unicode_type.pyx cython-0.20.1+1~202203241016-9537/tests/run/py_unicode_type.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/py_unicode_type.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/py_unicode_type.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,6 @@
 # -*- coding: iso-8859-1 -*-
+# mode: run
+# tag: warnings
 
 cimport cython
 
@@ -121,8 +123,8 @@
         uchar.isupper(),
         ]
 
-@cython.test_assert_path_exists('//PythonCapiCallNode')
-@cython.test_fail_if_path_exists('//SimpleCallNode')
+#@cython.test_assert_path_exists('//PythonCapiCallNode')
+#@cython.test_fail_if_path_exists('//SimpleCallNode')
 def unicode_methods(Py_UNICODE uchar):
     """
     >>> unicode_methods(ord('A')) == ['a', 'A', 'A']
@@ -229,3 +231,26 @@
     for i in range(1,9):
         if u'abcdefgh'[-i] in u'abCDefGh':
             print i
+
+
+def uchar_lookup_in_dict(obj, Py_UNICODE uchar):
+    """
+    >>> d = {u_KLINGON: 1234, u0: 0, u1: 1, u_A: 2}
+    >>> uchar_lookup_in_dict(d, u_KLINGON)
+    (1234, 1234)
+    >>> uchar_lookup_in_dict(d, u_A)
+    (2, 2)
+    >>> uchar_lookup_in_dict(d, u0)
+    (0, 0)
+    >>> uchar_lookup_in_dict(d, u1)
+    (1, 1)
+    """
+    cdef dict d = obj
+    dval = d[uchar]
+    objval = obj[uchar]
+    return dval, objval
+
+
+_WARNINGS = """
+250:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour.
+"""
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/raise_memory_error_T650.pyx cython-0.20.1+1~202203241016-9537/tests/run/raise_memory_error_T650.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/raise_memory_error_T650.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/raise_memory_error_T650.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 650
+# ticket: t650
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/range_optimisation_T203.pyx cython-0.20.1+1~202203241016-9537/tests/run/range_optimisation_T203.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/range_optimisation_T203.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/range_optimisation_T203.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 203
+# ticket: t203
 
 cdef int get_bound(int m):
     print u"get_bound(%s)"%m
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/r_docstrings.pyx cython-0.20.1+1~202203241016-9537/tests/run/r_docstrings.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/r_docstrings.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/r_docstrings.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -14,10 +14,21 @@
 
     >>> C.__doc__
     '\\n    This is a class docstring.\\n    '
+    >>> C.docstring_copy_C
+    '\\n    This is a class docstring.\\n    '
+    >>> CS.docstring_copy_C
+    '\\n    This is a class docstring.\\n    '
+
     >>> CS.__doc__
     '\\n    This is a subclass docstring.\\n    '
+    >>> CS.docstring_copy_CS
+    '\\n    This is a subclass docstring.\\n    '
+    >>> CSS.docstring_copy_CS
+    '\\n    This is a subclass docstring.\\n    '
     >>> print(CSS.__doc__)
     None
+    >>> CSS.docstring_copy_CSS
+    'A module docstring'
 
     >>> T.__doc__
     '\\n    This is an extension type docstring.\\n    '
@@ -34,22 +45,38 @@
     >>> Pyf.__doc__
     '\\n    This is a function docstring.\\n    '
 
-    >>> class PyC:
+    >>> class PyC(object):
     ...     '''
     ...     This is a class docstring.
     ...     '''
-    >>> class PyCS(C):
+    ...     docstring_copy_C = __doc__
+    >>> class PyCS(PyC):
     ...     '''
     ...     This is a subclass docstring.
     ...     '''
-    >>> class PyCSS(CS):
-    ...     pass
+    ...     docstring_copy_CS = __doc__
+    >>> class PyCSS(PyCS):
+    ...     docstring_copy_CSS = __doc__
 
     >>> PyC.__doc__
     '\\n    This is a class docstring.\\n    '
+    >>> PyC.docstring_copy_C
+    '\\n    This is a class docstring.\\n    '
+    >>> PyCS.docstring_copy_C
+    '\\n    This is a class docstring.\\n    '
+    >>> PyCSS.docstring_copy_C
+    '\\n    This is a class docstring.\\n    '
+
     >>> PyCS.__doc__
     '\\n    This is a subclass docstring.\\n    '
+    >>> PyCS.docstring_copy_CS
+    '\\n    This is a subclass docstring.\\n    '
+    >>> PyCSS.docstring_copy_CS
+    '\\n    This is a subclass docstring.\\n    '
+
     >>> PyCSS.__doc__
+    >>> PyCSS.docstring_copy_CSS
+    'A module docstring'
 """
 
 __test__ = {"test_docstrings" : doctest}
@@ -59,18 +86,24 @@
     This is a function docstring.
     """
 
-class C:
+
+class C(object):
     """
     This is a class docstring.
     """
+    docstring_copy_C = __doc__
+
 
 class CS(C):
     """
     This is a subclass docstring.
     """
+    docstring_copy_CS = __doc__
+
 
 class CSS(CS):
-    pass
+    docstring_copy_CSS = __doc__
+
 
 cdef class T:
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reduce_pickle.pyx cython-0.20.1+1~202203241016-9537/tests/run/reduce_pickle.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/reduce_pickle.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reduce_pickle.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,7 @@
+# mode: run
+# tag: pickle
+
+import cython
 import sys
 
 if sys.version_info[0] < 3:
@@ -7,7 +11,7 @@
     A(5)
     >>> cPickle.loads(cPickle.dumps(a))
     A(5)
-    
+
     >>> b = B(0, 1); b
     B(x=0, y=1)
     >>> cPickle.loads(cPickle.dumps(b))
@@ -45,15 +49,289 @@
 
     cdef int x, y
 
+    def __cinit__(self):
+        self.x = self.y = -1
+
     def __init__(self, x=0, y=0):
         self.x = x
         self.y = y
 
     def __repr__(self):
-        return "B(x=%s, y=%s)" % (self.x, self.y)
+        return "%s(x=%s, y=%s)" % (self.__class__.__name__, self.x, self.y)
 
     def __reduce__(self):
-        return makeB, ({'x': self.x, 'y': self.y},)
+        return makeObj, (type(self), {'x': self.x, 'y': self.y})
+
+def makeObj(obj_type, kwds):
+    return obj_type(**kwds)
+
+
+cdef class C(B):
+    """
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(C(x=37, y=389)))
+    C(x=37, y=389)
+    """
+    pass
+
+
+@cython.auto_pickle(True)  # Not needed, just to test the directive.
+cdef class DefaultReduce(object):
+    """
+    >>> a = DefaultReduce(11, 'abc'); a
+    DefaultReduce(i=11, s='abc')
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(a))
+    DefaultReduce(i=11, s='abc')
+    >>> pickle.loads(pickle.dumps(DefaultReduce(i=11, s=None)))
+    DefaultReduce(i=11, s=None)
+    """
+
+    cdef readonly int i
+    cdef readonly str s
+
+    def __init__(self, i=0, s=None):
+        self.i = i
+        self.s = s
+
+    def __repr__(self):
+        return "DefaultReduce(i=%s, s=%r)" % (self.i, self.s)
+
+
+cdef class DefaultReduceSubclass(DefaultReduce):
+    """
+    >>> a = DefaultReduceSubclass(i=11, s='abc', x=1.5); a
+    DefaultReduceSubclass(i=11, s='abc', x=1.5)
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(a))
+    DefaultReduceSubclass(i=11, s='abc', x=1.5)
+    """
+
+    cdef double x
+
+    def __init__(self, **kwargs):
+        self.x = kwargs.pop('x', 0)
+        super(DefaultReduceSubclass, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return "DefaultReduceSubclass(i=%s, s=%r, x=%s)" % (self.i, self.s, self.x)
+
+
+cdef class result(DefaultReduceSubclass):
+    """
+    >>> a = result(i=11, s='abc', x=1.5); a
+    result(i=11, s='abc', x=1.5)
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(a))
+    result(i=11, s='abc', x=1.5)
+    """
+
+    def __repr__(self):
+        return "result(i=%s, s=%r, x=%s)" % (self.i, self.s, self.x)
+
+
+class DefaultReducePySubclass(DefaultReduce):
+    """
+    >>> a = DefaultReducePySubclass(i=11, s='abc', x=1.5); a
+    DefaultReducePySubclass(i=11, s='abc', x=1.5)
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(a))
+    DefaultReducePySubclass(i=11, s='abc', x=1.5)
+
+    >>> a.self_reference = a
+    >>> a2 = pickle.loads(pickle.dumps(a))
+    >>> a2.self_reference is a2
+    True
+    """
+    def __init__(self, **kwargs):
+        self.x = kwargs.pop('x', 0)
+        super(DefaultReducePySubclass, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return "DefaultReducePySubclass(i=%s, s=%r, x=%s)" % (self.i, self.s, self.x)
+
+
+cdef class NoReduceDueToIntPtr(object):
+    """
+    >>> import pickle
+    >>> pickle.dumps(NoReduceDueToIntPtr())
+    Traceback (most recent call last):
+    ...
+    TypeError: self.int_ptr cannot be converted to a Python object for pickling
+    """
+    cdef int* int_ptr
+
+cdef class NoReduceDueToNontrivialCInit(object):
+    """
+    >>> import pickle
+    >>> pickle.dumps(NoReduceDueToNontrivialCInit(None))
+    Traceback (most recent call last):
+    ...
+    TypeError: no default __reduce__ due to non-trivial __cinit__
+    """
+    def __cinit__(self, arg):
+        pass
+
+
+cdef class NoMembers(object):
+    """
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(NoMembers()))
+    NoMembers()
+    """
+    def __repr__(self):
+        return "NoMembers()"
+
+
+cdef class NoPyMembers(object):
+    """
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(NoPyMembers(2, 1.75)))
+    NoPyMembers(ii=[2, 4, 8], x=1.75)
+    """
+    cdef int[3] ii
+    cdef double x
+
+    def __init__(self, i, x):
+        self.ii[0] = i
+        self.ii[1] = i * i
+        self.ii[2] = i * i * i
+        self.x = x
+
+    def __repr__(self):
+        return "%s(ii=%s, x=%s)" % (type(self).__name__, self.ii, self.x)
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, NoPyMembers) and
+            ( other).ii[0] == self.ii[0] and
+            ( other).ii[1] == self.ii[1] and
+            ( other).ii[2] == self.ii[2] and
+            ( other).x == self.x
+        )
+
+
+class NoPyMembersPySubclass(NoPyMembers):
+    """
+    >>> import pickle
+    >>> pickle.loads(pickle.dumps(NoPyMembersPySubclass(2, 1.75, 'xyz')))
+    NoPyMembersPySubclass(ii=[2, 4, 8], x=1.75, s='xyz')
+    """
+    def __init__(self, i, x, s):
+        super(NoPyMembersPySubclass, self).__init__(i, x)
+        self.s = s
+    def __repr__(self):
+        return (super(NoPyMembersPySubclass, self).__repr__()
+                [:-1] + ', s=%r)' % self.s)
+
+
+cdef struct MyStruct:
+    int i
+    double x
+
+cdef class StructMemberDefault(object):
+    """
+    >>> import pickle
+    >>> s = StructMemberDefault(1, 1.5); s
+    StructMemberDefault(i=1, x=1.5)
+    >>> pickle.dumps(s)   # doctest: +ELLIPSIS
+    Traceback (most recent call last):
+    TypeError: ...my_struct...
+    """
+
+    cdef MyStruct my_struct
+
+    def __init__(self, i, x):
+        self.my_struct.i = i
+        self.my_struct.x = x
+
+    def __repr__(self):
+        return "%s(i=%s, x=%s)" % (
+            type(self).__name__, self.my_struct.i, self.my_struct.x)
+
+@cython.auto_pickle(True)  # Forced due to the (inherited) struct attribute.
+cdef class StructMemberForcedPickle(StructMemberDefault):
+    """
+    >>> import pickle
+    >>> s = StructMemberForcedPickle(1, 1.5); s
+    StructMemberForcedPickle(i=1, x=1.5)
+    >>> pickle.loads(pickle.dumps(s))
+    StructMemberForcedPickle(i=1, x=1.5)
+    """
+
+
+cdef _unset = object()
 
-def makeB(kwds):
-    return B(**kwds)
+# Test cyclic references.
+cdef class Wrapper(object):
+  """
+  >>> import pickle
+  >>> w = Wrapper(); w
+  Wrapper(...)
+  >>> w2 = pickle.loads(pickle.dumps(w)); w2
+  Wrapper(...)
+  >>> w2.ref is w2
+  True
+
+  >>> pickle.loads(pickle.dumps(Wrapper(DefaultReduce(1, 'xyz'))))
+  Wrapper(DefaultReduce(i=1, s='xyz'))
+  >>> L = [None]
+  >>> L[0] = L
+  >>> w = Wrapper(L)
+  >>> pickle.loads(pickle.dumps(Wrapper(L)))
+  Wrapper([[...]])
+
+  >>> L[0] = w   # Don't print this one out...
+  >>> w2 = pickle.loads(pickle.dumps(w))
+  >>> w2.ref[0] is w2
+  True
+  """
+  cdef public object ref
+  def __init__(self, ref=_unset):
+      if ref is _unset:
+          self.ref = self
+      else:
+          self.ref = ref
+  def __repr__(self):
+      if self.ref is self:
+          return "Wrapper(...)"
+      else:
+          return "Wrapper(%r)" % self.ref
+
+
+# Non-regression test for pickling bound and unbound methods of non-extension
+# classes
+if sys.version_info[:2] >= (3, 5):
+    # builtin methods not picklable for python <= 3.4
+    class MyClass(object):
+        """
+        >>> import pickle
+        >>> pickle.loads(pickle.dumps(MyClass.my_method)) is MyClass.my_method
+        True
+        >>> bound_method = pickle.loads(pickle.dumps(MyClass().my_method))
+        >>> bound_method(1)
+        1
+        """
+        def my_method(self, x):
+            return x
+
+
+# Pickled with Cython 0.29.28 (using MD5 for the checksum).
+OLD_MD5_PICKLE = b'''\
+creduce_pickle\n__pyx_unpickle_NoPyMembers\nq\x00\
+(creduce_pickle\nNoPyMembers\nq\x01J\xf2K_\n(]q\x02\
+(K\x0bKyM3\x05eG?\xf8\x00\x00\x00\x00\x00\x00tq\x03tq\x04Rq\x05.\
+'''
+
+try:
+    from hashlib import md5
+except ImportError:
+    pass
+else:
+    def unpickle_old_0_29_28():
+        """
+        >>> import pickle
+        >>> b = pickle.loads(OLD_MD5_PICKLE)
+        >>> b == NoPyMembers(i=11, x=1.5) or b
+        True
+        """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/refcount_in_meth.pyx cython-0.20.1+1~202203241016-9537/tests/run/refcount_in_meth.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/refcount_in_meth.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/refcount_in_meth.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -12,8 +12,10 @@
 True
 """
 
+cimport cython
 from cpython.ref cimport PyObject
 
+@cython.always_allow_keywords(False)
 def get_refcount(obj):
     return (obj).ob_refcnt
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reimport_failure.srctree cython-0.20.1+1~202203241016-9537/tests/run/reimport_failure.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/reimport_failure.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reimport_failure.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,38 @@
+# mode: run
+# tag: pep489
+
+"""
+PYTHON setup.py build_ext -i
+PYTHON tester.py
+"""
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("*.pyx"),
+)
+
+
+######## failure.pyx ########
+
+if globals():  # runtime True to confuse dead code removal
+    raise ImportError
+
+cdef class C:
+    cdef int a
+
+
+######## tester.py ########
+
+try:
+    try:
+        import failure  # 1
+    except ImportError:
+        import failure  # 2
+except ImportError:
+    pass
+else:
+    raise RuntimeError("ImportError was not raised on second import!")
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reimport_from_package.srctree cython-0.20.1+1~202203241016-9537/tests/run/reimport_from_package.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/reimport_from_package.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reimport_from_package.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,6 @@
+# mode: run
+# tag: pep489
+
 PYTHON setup.py build_ext --inplace
 PYTHON -c "import a"
 
@@ -14,8 +17,8 @@
 
 import sys
 import a
-assert a in sys.modules.values(), list(sys.modules)
-assert sys.modules['a'] is a, list(sys.modules)
+assert a in sys.modules.values(), sorted(sys.modules)
+assert sys.modules['a'] is a, sorted(sys.modules)
 
 from atest.package import module
 
@@ -30,5 +33,27 @@
 
 import a
 import atest.package.module as module
-assert module in sys.modules.values(), list(sys.modules)
-assert sys.modules['atest.package.module'] is module, list(sys.modules)
+assert module in sys.modules.values(), sorted(sys.modules)
+assert sys.modules['atest.package.module'] is module, sorted(sys.modules)
+
+if sys.version_info >= (3, 5):
+    from . import pymodule
+    assert module is pymodule.import_without_package()
+
+######## atest/package/pymodule.py ########
+
+from . import module
+from ..package import module
+import atest.package.module
+
+import a
+
+def import_without_package():
+    import os.path
+    import sys
+    sys.path.insert(0, os.path.dirname(__file__))
+    sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
+    import module
+    import package.module
+    assert package.module is module
+    return module
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reimport_from_subinterpreter.srctree cython-0.20.1+1~202203241016-9537/tests/run/reimport_from_subinterpreter.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/reimport_from_subinterpreter.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reimport_from_subinterpreter.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,82 @@
+# mode: run
+# tag: pep489, subinterpreter
+
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import subtest; subtest.run_main()"
+PYTHON -c "import subtest; subtest.run_sub()"
+PYTHON -c "import subtest; subtest.run_main(); subtest.run_sub()"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(
+    ext_modules = cythonize("**/*.pyx"),
+    )
+
+######## subtest.pyx ########
+
+cdef extern from *:
+    """
+    /* Copied from CPython's _testcapi.c module */
+    static PyObject *run_in_subinterpreter(const char *code) {
+        int r;
+        PyThreadState *substate, *mainstate;
+
+        mainstate = PyThreadState_Get();
+
+        PyThreadState_Swap(NULL);
+
+        substate = Py_NewInterpreter();
+        if (substate == NULL) {
+            /* Since no new thread state was created, there is no exception to
+               propagate; raise a fresh one after swapping in the old thread
+               state. */
+            PyThreadState_Swap(mainstate);
+            PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed");
+            return NULL;
+        }
+        r = PyRun_SimpleString(code);
+        Py_EndInterpreter(substate);
+
+        PyThreadState_Swap(mainstate);
+        return PyLong_FromLong(r);
+    }
+    """
+    object run_in_subinterpreter(const char *code)
+
+
+MAIN_HAS_IMPORTED = False
+
+def run_main():
+    global MAIN_HAS_IMPORTED
+    MAIN_HAS_IMPORTED = True
+    import package.subtest
+    from package import subtest
+
+def run_sub():
+    assert 0 == run_in_subinterpreter(b'1+1')
+    assert 0 == run_in_subinterpreter(b'2+2')
+
+    # The subinterpreter does not add the current working directory to
+    # sys.path, so we need to add it manually.
+    pre = b'import sys; sys.path.insert(0, "."); '
+    assert 0 == run_in_subinterpreter(pre + b'import package')
+    assert 0 == run_in_subinterpreter(pre + b'import package')
+
+    import sys
+    result = run_in_subinterpreter(pre + b'import package.subtest')
+    if not MAIN_HAS_IMPORTED:
+        assert result == 0, result  # imports only in subinterpreters are ok
+    elif sys.version_info >= (3, 5):
+        assert result == -1, result  # re-import in a different subinterpreter fails in Py3.5+ (with PEP-489)
+    else:
+        assert result == 0, result  # re-import in a different subinterpreter reuses the module in Py<3.5
+
+
+######## package/__init__.py ########
+
+######## package/subtest.pyx ########
+
+print("Module loaded: %s" % __name__)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/relative_cimport.srctree cython-0.20.1+1~202203241016-9537/tests/run/relative_cimport.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/relative_cimport.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/relative_cimport.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -3,6 +3,7 @@
 
 PYTHON setup.py build_ext --inplace
 PYTHON -c "from pkg.b import test; assert test() == (1, 2)"
+PYTHON -c "from pkg.b_py2 import test; assert test() == (1, 2)"
 PYTHON -c "from pkg.sub.c import test; assert test() == (1, 2)"
 
 ######## setup.py ########
@@ -42,7 +43,23 @@
 
 from . cimport a
 from .a cimport test_pxd
-cimport a as implicitly_relative_a
+
+assert a.test_pxd is test_pxd
+
+def test():
+    cdef test_pxd obj = test_pxd()
+    obj.x = 1
+    obj.y = 2
+    return (obj.x, obj.y)
+
+
+######## pkg/b_py2.pyx ########
+
+# cython: language_level=2
+
+from . cimport a
+from .a cimport test_pxd
+cimport a as implicitly_relative_a  # <-- Py2 "feature"
 
 assert a.test_pxd is test_pxd
 assert implicitly_relative_a.test_pxd is test_pxd
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/relativeimport_star_T542.pyx cython-0.20.1+1~202203241016-9537/tests/run/relativeimport_star_T542.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/relativeimport_star_T542.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/relativeimport_star_T542.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,8 @@
 from distutils import core, version
+
 __name__='distutils.core.cytest_relativeimport_T542' # fool Python we are in distutils
+__package__='distutils.core' # fool Python we are in distutils
+
 from . import *
 
 __doc__ = """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/relativeimport_T542.srctree cython-0.20.1+1~202203241016-9537/tests/run/relativeimport_T542.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/relativeimport_T542.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/relativeimport_T542.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -26,7 +26,8 @@
 except ImportError:
     pass
 else:
-    assert False, "absolute import succeeded"
+    import sys
+    assert False, "absolute import succeeded: %s" % sorted(sys.modules)
 
 import relimport.a
 import relimport.bmod
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reload_ext_module.pyx cython-0.20.1+1~202203241016-9537/tests/run/reload_ext_module.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/reload_ext_module.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reload_ext_module.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,16 @@
+# mode: run
+# tag: pep489, no-macos
+
+# FIXME: don't know why this does not work on MacOS, but it's not worth failing the builds for now.
+
+
+import reload_ext_module
+
+
+def test_reload(module):
+    """
+    >>> module = test_reload(reload_ext_module)
+    >>> module is reload_ext_module  # Py_mod_create enforces a singleton.
+    True
+    """
+    return reload(module)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reraise.py cython-0.20.1+1~202203241016-9537/tests/run/reraise.py
--- cython-0.20.1+1~201611251650-6686/tests/run/reraise.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reraise.py	2022-03-24 10:16:46.000000000 +0000
@@ -31,6 +31,6 @@
     ... else: print("FAILED")
     """
     import sys
-    if hasattr(sys, 'exc_clear'): # Py2
+    if hasattr(sys, 'exc_clear'):  # Py2
         sys.exc_clear()
     raise
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/reversed_iteration.pyx cython-0.20.1+1~202203241016-9537/tests/run/reversed_iteration.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/reversed_iteration.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/reversed_iteration.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -308,9 +308,9 @@
 
 def reversed_range_step3_py_obj_left(a, int b):
     """
-    >>> reversed_range_step3_py_obj_left(set(), 0)
+    >>> reversed_range_step3_py_obj_left(set(), 0)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef long i
     result = []
@@ -319,9 +319,9 @@
 
 def reversed_range_step3_py_obj_right(int a, b):
     """
-    >>> reversed_range_step3_py_obj_right(0, set())
+    >>> reversed_range_step3_py_obj_right(0, set())  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef long i
     result = []
@@ -330,9 +330,9 @@
 
 def reversed_range_step3_neg_py_obj_left(a, int b):
     """
-    >>> reversed_range_step3_neg_py_obj_left(set(), 0)
+    >>> reversed_range_step3_neg_py_obj_left(set(), 0)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef long i
     result = []
@@ -341,9 +341,9 @@
 
 def reversed_range_step3_neg_py_obj_right(int a, b):
     """
-    >>> reversed_range_step3_py_obj_right(0, set())
+    >>> reversed_range_step3_py_obj_right(0, set())  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef long i
     result = []
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/r_forloop.pyx cython-0.20.1+1~202203241016-9537/tests/run/r_forloop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/r_forloop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/r_forloop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -71,7 +71,7 @@
     Spam!
     Spam!
     """
-    cdef int i
+    cdef long i
     for i in range(4*x,2*x,-3):
         print u"Spam!"
 
@@ -94,7 +94,7 @@
     Spam!
     Spam!
     """
-    cdef int i
+    cdef long i
     for i in range(2*f(x),f(x), -2):
         print u"Spam!"
 
@@ -103,7 +103,7 @@
     >>> go_c_calc_ret(2)
     6
     """
-    cdef int i
+    cdef long i
     for i in range(2*f(x),f(x), -2):
         if i < 2*f(x):
             return i
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/scanner_trace.srctree cython-0.20.1+1~202203241016-9537/tests/run/scanner_trace.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/scanner_trace.srctree	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/scanner_trace.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -9,11 +9,26 @@
 
 Cython.Compiler.Scanning.trace_scanner = 1
 
-setup(
-    ext_modules = cythonize("*.pyx")
-)
+setup(ext_modules=cythonize("*.pyx"))
+
+try:
+    from importlib.util import spec_from_file_location, module_from_spec
+except ImportError:
+    # Py<=3.4
+    # Try to import from the current directory.
+    import os, sys
+    sys.path.insert(0, os.getcwd())
+    import simple
+else:
+    # Py3.5+
+    import glob
+    ext_files = glob.glob("simple*.so") + glob.glob("simple*.pyd")
+    assert ext_files
+    spec = spec_from_file_location('simple', ext_files[0])
+    simple = module_from_spec(spec)
+    spec.loader.exec_module(simple)
+
 
-import simple
 assert simple.test() == 123
 
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/self_in_ext_type_closure.pyx cython-0.20.1+1~202203241016-9537/tests/run/self_in_ext_type_closure.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/self_in_ext_type_closure.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/self_in_ext_type_closure.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 742
+# ticket: t742
 
 import cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/seq_mul.py cython-0.20.1+1~202203241016-9537/tests/run/seq_mul.py
--- cython-0.20.1+1~201611251650-6686/tests/run/seq_mul.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/seq_mul.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,148 @@
+# mode: run
+# tag: list, mulop, pure3.0
+
+import cython
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+@cython.test_assert_path_exists("//ListNode[@mult_factor]")
+def cint_times_list(n: cython.int):
+    """
+    >>> cint_times_list(3)
+    []
+    [None, None, None]
+    [3, 3, 3]
+    [1, 2, 3, 1, 2, 3, 1, 2, 3]
+    """
+    a = n * []
+    b = n * [None]
+    c = n * [n]
+    d = n * [1, 2, 3]
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+@cython.test_assert_path_exists("//ListNode[@mult_factor]")
+def list_times_cint(n: cython.int):
+    """
+    >>> list_times_cint(3)
+    []
+    [None, None, None]
+    [3, 3, 3]
+    [1, 2, 3, 1, 2, 3, 1, 2, 3]
+    """
+    a = [] * n
+    b = [None] * n
+    c = [n] * n
+    d = [1, 2, 3] * n
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+@cython.test_assert_path_exists("//TupleNode[@mult_factor]")
+def cint_times_tuple(n: cython.int):
+    """
+    >>> cint_times_tuple(3)
+    ()
+    (None, None, None)
+    (3, 3, 3)
+    (1, 2, 3, 1, 2, 3, 1, 2, 3)
+    """
+    a = n * ()
+    b = n * (None,)
+    c = n * (n,)
+    d = n * (1, 2, 3)
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+@cython.test_assert_path_exists("//TupleNode[@mult_factor]")
+def tuple_times_cint(n: cython.int):
+    """
+    >>> tuple_times_cint(3)
+    ()
+    (None, None, None)
+    (3, 3, 3)
+    (1, 2, 3, 1, 2, 3, 1, 2, 3)
+    """
+    a = () * n
+    b = (None,) * n
+    c = (n,) * n
+    d = (1, 2, 3) * n
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
+
+
+# TODO: enable in Cython 3.1 when we can infer unsafe C int operations as PyLong
+#@cython.test_fail_if_path_exists("//MulNode")
+def list_times_pyint(n: cython.longlong):
+    """
+    >>> list_times_cint(3)
+    []
+    [None, None, None]
+    [3, 3, 3]
+    [1, 2, 3, 1, 2, 3, 1, 2, 3]
+    """
+    py_n = n + 1  # might overflow => should be inferred as Python long!
+
+    a = [] * py_n
+    b = [None] * py_n
+    c = py_n * [n]
+    d = py_n * [1, 2, 3]
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
+
+
+@cython.cfunc
+def sideeffect(x) -> cython.int:
+    global _sideeffect_value
+    _sideeffect_value += 1
+    return _sideeffect_value + x
+
+
+def reset_sideeffect():
+    global _sideeffect_value
+    _sideeffect_value = 0
+
+
+@cython.test_fail_if_path_exists("//MulNode")
+@cython.test_assert_path_exists("//ListNode[@mult_factor]")
+def complicated_cint_times_list(n: cython.int):
+    """
+    >>> complicated_cint_times_list(3)
+    []
+    [None, None, None, None]
+    [3, 3, 3, 3]
+    [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3]
+    """
+    reset_sideeffect()
+    a = [] * sideeffect((lambda: n)())
+    reset_sideeffect()
+    b = sideeffect((lambda: n)()) * [None]
+    reset_sideeffect()
+    c = [n] * sideeffect((lambda: n)())
+    reset_sideeffect()
+    d = sideeffect((lambda: n)()) * [1, 2, 3]
+
+    print(a)
+    print(b)
+    print(c)
+    print(d)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/sequential_parallel.pyx cython-0.20.1+1~202203241016-9537/tests/run/sequential_parallel.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/sequential_parallel.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/sequential_parallel.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,6 +2,7 @@
 
 cimport cython.parallel
 from cython.parallel import prange, threadid
+from cython.view cimport array
 from libc.stdlib cimport malloc, calloc, free, abort
 from libc.stdio cimport puts
 
@@ -59,7 +60,7 @@
     >>> test_prange_matches_range(2, -10, -3)
     >>> test_prange_matches_range(3, -10, -3)
     """
-    cdef int i, range_last, prange_last
+    cdef int i = -765432, range_last = -876543, prange_last = -987654
     prange_set = set()
     for i in prange(start, stop, step, nogil=True, num_threads=3):
         prange_last = i
@@ -377,11 +378,7 @@
 
 def test_nested_break_continue():
     """
-    DISABLED. For some reason this fails intermittently on jenkins, with
-    the first line of output being '0 0 0 0'. The generated code looks
-    awfully correct though... needs investigation
-
-    >> test_nested_break_continue()
+    >>> test_nested_break_continue()
     6 7 6 7
     8
     """
@@ -659,7 +656,7 @@
         sum += inner_parallel_section()
     return sum
 
-cdef int nogil_cdef_except_clause() nogil except 0:
+cdef int nogil_cdef_except_clause() nogil except -1:
     return 1
 
 cdef void nogil_cdef_except_star() nogil except *:
@@ -737,3 +734,85 @@
     except Exception, e:
         print e.args[0]
 
+
+def test_pointer_temps(double x):
+    """
+    >>> test_pointer_temps(1.0)
+    4.0
+    """
+    cdef Py_ssize_t i
+    cdef double* f
+    cdef double[:] arr = array(format="d", shape=(10,), itemsize=sizeof(double))
+    arr[0] = 4.0
+    arr[1] = 3.0
+
+    for i in prange(10, nogil=True, num_threads=1):
+        f = &arr[0]
+
+    return f[0]
+
+
+def test_prange_in_with(int x, ctx):
+    """
+    >>> from contextlib import contextmanager
+    >>> @contextmanager
+    ... def ctx(l): yield l
+    >>> test_prange_in_with(4, ctx([0]))
+    6
+    """
+    cdef int i
+    with ctx as l:
+        for i in prange(x, nogil=True):
+            with gil:
+                l[0] += i
+        return l[0]
+
+
+cdef extern from *:
+    """
+    #ifdef _OPENMP
+    #define _get_addr(_x, _idx) &_x
+    #else
+    #define _get_addr(_x, _idx) (&_x+_idx)
+    #endif
+    #define address_of_temp(store, temp, idx) store = _get_addr(temp, idx)
+    #define address_of_temp2(store, ignore, temp, idx) store = _get_addr(temp, idx)
+
+    double get_value() {
+        return 1.0;
+    }
+    """
+    void address_of_temp(...) nogil
+    void address_of_temp2(...) nogil
+    double get_value() nogil except -1.0  # will generate a temp for exception checking
+
+def test_inner_private():
+    """
+    Determines if a temp variable is private by taking its address in multiple threads
+    and seeing if they're the same (thread private variables should have different
+    addresses
+    >>> test_inner_private()
+    ok
+    """
+    cdef double* not_parallel[2]
+    cdef double* inner_vals[2]
+    cdef double* outer_vals[2]
+    cdef Py_ssize_t n, m
+
+    for n in range(2):
+        address_of_temp(not_parallel[n], get_value(), 0)
+    assert not_parallel[0] == not_parallel[1], "Addresses should be the same since they come from the same temp"
+
+    for n in prange(2, num_threads=2, schedule='static', chunksize=1, nogil=True):
+        address_of_temp(outer_vals[n], get_value(), n)
+        for m in prange(1):
+            # second temp just ensures different numbering
+            address_of_temp2(inner_vals[n], get_value(), get_value(), n)
+
+    inner_are_the_same = inner_vals[0] == inner_vals[1]
+    outer_are_the_same = outer_vals[0] == outer_vals[1]
+
+    assert outer_are_the_same == False, "Temporary variables in outer loop should be private"
+    assert inner_are_the_same == False,  "Temporary variables in inner loop should be private"
+
+    print('ok')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/set_item.pyx cython-0.20.1+1~202203241016-9537/tests/run/set_item.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/set_item.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/set_item.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,75 @@
+# mode: run
+# tag: list, dict, setitem, delitem
+
+def set_item(obj, key, value):
+    """
+    >>> set_item([1, 2, 3], 1, -1)
+    [1, -1, 3]
+    >>> set_item([1, 2, 3], -1, -1)
+    [1, 2, -1]
+    >>> set_item({}, 'abc', 5)
+    {'abc': 5}
+    >>> set_item({}, -1, 5)
+    {-1: 5}
+    >>> class D(dict): pass
+    >>> set_item(D({}), 'abc', 5)
+    {'abc': 5}
+    >>> set_item(D({}), -1, 5)
+    {-1: 5}
+    """
+    obj[key] = value
+    return obj
+
+
+def set_item_int(obj, int key, value):
+    """
+    >>> set_item_int([1, 2, 3], 1, -1)
+    [1, -1, 3]
+    >>> set_item_int([1, 2, 3], -1, -1)
+    [1, 2, -1]
+    >>> set_item_int({}, 1, 5)
+    {1: 5}
+    >>> set_item_int({}, -1, 5)
+    {-1: 5}
+    >>> class D(dict): pass
+    >>> set_item_int(D({}), 1, 5)
+    {1: 5}
+    >>> set_item_int(D({}), -1, 5)
+    {-1: 5}
+    """
+    obj[key] = value
+    return obj
+
+
+def del_item(obj, key):
+    """
+    >>> del_item([1, 2, 3], 1)
+    [1, 3]
+    >>> del_item([1, 2, 3], -3)
+    [2, 3]
+    >>> class D(dict): pass
+    >>> del_item({'abc': 1, 'def': 2}, 'abc')
+    {'def': 2}
+    >>> del_item(D({'abc': 1, 'def': 2}), 'abc')
+    {'def': 2}
+    >>> del_item(D({-1: 1, -2: 2}), -1)
+    {-2: 2}
+    """
+    del obj[key]
+    return obj
+
+
+def del_item_int(obj, int key):
+    """
+    >>> del_item_int([1, 2, 3], 1)
+    [1, 3]
+    >>> del_item_int([1, 2, 3], -3)
+    [2, 3]
+    >>> class D(dict): pass
+    >>> del_item_int(D({-1: 1, 1: 2}), 1)
+    {-1: 1}
+    >>> del_item_int(D({-1: 1, -2: 2}), -1)
+    {-2: 2}
+    """
+    del obj[key]
+    return obj
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/set_iter.pyx cython-0.20.1+1~202203241016-9537/tests/run/set_iter.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/set_iter.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/set_iter.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,99 @@
+# mode: run
+# tag: set
+
+cimport cython
+
+
+@cython.test_assert_path_exists(
+    "//SetIterationNextNode",
+)
+def set_iter_comp(set s):
+    """
+    >>> s = set([1, 2, 3])
+    >>> sorted(set_iter_comp(s))
+    [1, 2, 3]
+    """
+    return [x for x in s]
+
+
+@cython.test_assert_path_exists(
+    "//SetIterationNextNode",
+)
+def set_iter_comp_typed(set s):
+    """
+    >>> s = set([1, 2, 3])
+    >>> sorted(set_iter_comp(s))
+    [1, 2, 3]
+    """
+    cdef int x
+    return [x for x in s]
+
+
+@cython.test_assert_path_exists(
+    "//SetIterationNextNode",
+)
+def frozenset_iter_comp(frozenset s):
+    """
+    >>> s = frozenset([1, 2, 3])
+    >>> sorted(frozenset_iter_comp(s))
+    [1, 2, 3]
+    """
+    return [x for x in s]
+
+
+@cython.test_assert_path_exists(
+    "//SetIterationNextNode",
+)
+def set_iter_comp_frozenset(set s):
+    """
+    >>> s = set([1, 2, 3])
+    >>> sorted(set_iter_comp(s))
+    [1, 2, 3]
+    """
+    return [x for x in frozenset(s)]
+
+
+@cython.test_assert_path_exists(
+    "//SetIterationNextNode",
+)
+def set_iter_modify(set s, int value):
+    """
+    >>> s = set([1, 2, 3])
+    >>> sorted(set_iter_modify(s, 1))
+    [1, 2, 3]
+    >>> sorted(set_iter_modify(s, 2))
+    [1, 2, 3]
+    >>> sorted(set_iter_modify(s, 3))
+    [1, 2, 3]
+    >>> sorted(set_iter_modify(s, 4))
+    Traceback (most recent call last):
+    RuntimeError: set changed size during iteration
+    """
+    for x in s:
+        s.add(value)
+    return s
+
+
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode//NameNode[@name = 'enumerate']",
+)
+@cython.test_assert_path_exists(
+    "//AddNode",
+    "//SetIterationNextNode",
+)
+def set_iter_enumerate(set s):
+    """
+    >>> s = set(['a', 'b', 'c'])
+    >>> numbers, values = set_iter_enumerate(s)
+    >>> sorted(numbers)
+    [0, 1, 2]
+    >>> sorted(values)
+    ['a', 'b', 'c']
+    """
+    cdef int i
+    numbers = []
+    values = []
+    for i, x in enumerate(s):
+        numbers.append(i)
+        values.append(x)
+    return numbers, values
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/set_new.py cython-0.20.1+1~202203241016-9537/tests/run/set_new.py
--- cython-0.20.1+1~201611251650-6686/tests/run/set_new.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/set_new.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,21 @@
+"""
+>>> X = make_class_with_new(cynew)
+>>> X.__new__ is cynew
+True
+>>> X().__new__ is cynew
+True
+>>> def pynew(cls): return object.__new__(cls)
+>>> X = make_class_with_new(pynew)
+>>> X.__new__ is pynew
+True
+>>> X().__new__ is pynew
+True
+"""
+
+def make_class_with_new(n):
+    class X(object):
+        __new__ = n
+    return X
+
+def cynew(cls):
+    return object.__new__(cls)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/set.pyx cython-0.20.1+1~202203241016-9537/tests/run/set.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/set.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/set.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -65,6 +65,32 @@
     return s1
 
 
+def test_set_contains(v):
+    """
+    >>> test_set_contains(1)
+    True
+    >>> test_set_contains(2)
+    False
+    >>> test_set_contains(frozenset([1, 2, 3]))
+    True
+    >>> test_set_contains(frozenset([1, 2]))
+    False
+    >>> test_set_contains(set([1, 2, 3]))
+    True
+    >>> test_set_contains(set([1, 2]))
+    False
+    >>> try: test_set_contains([1, 2])
+    ... except TypeError: pass
+    ... else: print("NOT RAISED!")
+    """
+    cdef set s1
+    s1 = set()
+    s1.add(1)
+    s1.add('a')
+    s1.add(frozenset([1, 2, 3]))
+    return v in s1
+
+
 def test_set_update(v=None):
     """
     >>> type(test_set_update()) is set
@@ -88,6 +114,18 @@
     return s1
 
 
+def test_set_multi_update():
+    """
+    >>> type(test_set_multi_update()) is set
+    True
+    >>> sorted(test_set_multi_update())
+    ['a', 'b', 'c', 1, 2, 3]
+    """
+    cdef set s1 = set()
+    s1.update('abc', set([1, 3]), frozenset([1, 2]))
+    return s1
+
+
 def test_object_update(v=None):
     """
     >>> type(test_object_update()) is set
@@ -379,7 +417,8 @@
     True
     >>> len(s)
     0
-    >>> s is frozenset()   # singleton!
+    >>> import sys
+    >>> sys.version_info >= (3, 10) or s is frozenset()   # singleton (in Python < 3.10)!
     True
     """
     return frozenset()
@@ -392,7 +431,8 @@
 )
 def test_singleton_empty_frozenset():
     """
-    >>> test_singleton_empty_frozenset()  # from CPython's test_set.py
+    >>> import sys
+    >>> test_singleton_empty_frozenset() if sys.version_info < (3, 10) else 1  # from CPython's test_set.py
     1
     """
     f = frozenset()
@@ -400,7 +440,7 @@
            frozenset(), frozenset([]), frozenset(()), frozenset(''),
            frozenset(range(0)), frozenset(frozenset()),
            frozenset(f), f]
-    return len(set(map(id, efs)))
+    return len(set(map(id, efs)))  # note, only a singleton in Python <3.10
 
 
 def sorted(it):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/short_circuit_T404.pyx cython-0.20.1+1~202203241016-9537/tests/run/short_circuit_T404.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/short_circuit_T404.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/short_circuit_T404.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 404
+# ticket: t404
 
 cdef long foo(long x):
     print "foo(%s)" % x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/simpcall.pyx cython-0.20.1+1~202203241016-9537/tests/run/simpcall.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/simpcall.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/simpcall.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# mode: test
+# mode: run
 
 def f(x, y):
     x = y
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/slice2.pyx cython-0.20.1+1~202203241016-9537/tests/run/slice2.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/slice2.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/slice2.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# tag: slicing
+# tag: list, slice, slicing
 
 def test_full(seq):
     """
@@ -18,21 +18,24 @@
 
 def test_start(seq, start):
     """
-    >>> test_start([1,2,3,4], 2)
+    >>> l = [1,2,3,4]
+    >>> test_start(l, 2)
     [3, 4]
-    >>> test_start([1,2,3,4], 3)
+    >>> test_start(l, 3)
     [4]
-    >>> test_start([1,2,3,4], 4)
+    >>> test_start(l, 4)
     []
-    >>> test_start([1,2,3,4], 8)
+    >>> test_start(l, 8)
     []
-    >>> test_start([1,2,3,4], -3)
+    >>> test_start(l, -3)
     [2, 3, 4]
-    >>> test_start([1,2,3,4], -4)
+    >>> test_start(l, -4)
+    [1, 2, 3, 4]
+    >>> test_start(l, -8)
     [1, 2, 3, 4]
-    >>> test_start([1,2,3,4], -8)
+    >>> test_start(l, 0)
     [1, 2, 3, 4]
-    >>> test_start([1,2,3,4], 0)
+    >>> test_start(l, None)
     [1, 2, 3, 4]
     >>> try: test_start(42, 2, 3)
     ... except TypeError: pass
@@ -42,24 +45,52 @@
 
 def test_stop(seq, stop):
     """
-    >>> test_stop([1,2,3,4], 3)
+    >>> l = [1,2,3,4]
+    >>> test_stop(l, 3)
     [1, 2, 3]
-    >>> test_stop([1,2,3,4], -1)
+    >>> test_stop(l, -1)
     [1, 2, 3]
-    >>> test_stop([1,2,3,4], -3)
+    >>> test_stop(l, -3)
     [1]
-    >>> test_stop([1,2,3,4], -4)
+    >>> test_stop(l, -4)
     []
-    >>> test_stop([1,2,3,4], -8)
+    >>> test_stop(l, -8)
     []
-    >>> test_stop([1,2,3,4], 0)
+    >>> test_stop(l, 0)
     []
+    >>> test_stop(l, None)
+    [1, 2, 3, 4]
     >>> try: test_stop(42, 3)
     ... except TypeError: pass
     """
     obj = seq[:stop]
     return obj
 
+def test_step(seq, step):
+    """
+    >>> l = [1,2,3,4]
+    >>> test_step(l, -1)
+    [4, 3, 2, 1]
+    >>> test_step(l, 1)
+    [1, 2, 3, 4]
+    >>> test_step(l, 2)
+    [1, 3]
+    >>> test_step(l, 3)
+    [1, 4]
+    >>> test_step(l, -3)
+    [4, 1]
+    >>> test_step(l, None)
+    [1, 2, 3, 4]
+    >>> try: test_step(l, 0)
+    ... except ValueError: pass
+    ...
+    >>> try: test_step(42, 0)
+    ... except TypeError: pass
+    ...
+    """
+    obj = seq[::step]
+    return obj
+
 def test_start_and_stop(seq, start, stop):
     """
     >>> l = [1,2,3,4]
@@ -67,12 +98,38 @@
     [3]
     >>> test_start_and_stop(l, -3, -1)
     [2, 3]
+    >>> test_start_and_stop(l, None, None)
+    [1, 2, 3, 4]
     >>> try: test_start_and_stop(42, 2, 3)
     ... except TypeError: pass
     """
     obj = seq[start:stop]
     return obj
 
+def test_start_stop_and_step(seq, start, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> test_start_stop_and_step(l, 0, 5, 1)
+    [1, 2, 3, 4, 5]
+    >>> test_start_stop_and_step(l, 5, -1, -1)
+    []
+    >>> test_start_stop_and_step(l, 5, None, -1)
+    [5, 4, 3, 2, 1]
+    >>> test_start_stop_and_step(l, 2, 5, 2)
+    [3, 5]
+    >>> test_start_stop_and_step(l, -100, 100, 1)
+    [1, 2, 3, 4, 5]
+    >>> test_start_stop_and_step(l, None, None, None)
+    [1, 2, 3, 4, 5]
+    >>> try: test_start_stop_and_step(l, None, None, 0)
+    ... except ValueError: pass
+    ... 
+    >>> try: test_start_stop_and_step(42, 1, 2, 3)
+    ... except TypeError: pass
+    """
+    obj = seq[start:stop:step]
+    return obj
+
 class A(object):
     pass
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561_py2.pyx cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561_py2.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561_py2.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561_py2.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 561
+# ticket: t561
 # tag: py2
 # This file tests the behavior of special methods under Python 2
 # after #561.  (Only methods whose behavior differs between Python 2 and 3
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561_py3.pyx cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561_py3.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561_py3.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561_py3.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 561
+# ticket: t561
 # tag: py3
 # This file tests the behavior of special methods under Python 3
 # after #561.  (Only methods whose behavior differs between Python 2 and 3
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561.pyx cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/special_methods_T561.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/special_methods_T561.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
-# ticket: 561
-# ticket: 3
+# ticket: t561
+# ticket: t3
 
 # The patch in #561 changes code generation for most special methods
 # to remove the Cython-generated wrapper and let PyType_Ready()
@@ -56,30 +56,6 @@
     >>> g11 = object.__getattribute__(GetAttribute(), '__getattribute__')
     >>> g11('attr')
     GetAttribute getattribute 'attr'
-    >>> # If you define either setattr or delattr, you get wrapper objects
-    >>> # for both methods.  (This behavior is unchanged by #561.)
-    >>> sa_setattr = SetAttr().__setattr__
-    >>> sa_setattr('foo', 'bar')
-    SetAttr setattr 'foo' 'bar'
-    >>> sa_delattr = SetAttr().__delattr__
-    >>> sa_delattr('foo')
-    Traceback (most recent call last):
-    ...
-    AttributeError: 'special_methods_T561.SetAttr' object has no attribute 'foo'
-    >>> da_setattr = DelAttr().__setattr__
-    >>> da_setattr('foo', 'bar')
-    Traceback (most recent call last):
-    ...
-    AttributeError: 'special_methods_T561.DelAttr' object has no attribute 'foo'
-    >>> da_delattr = DelAttr().__delattr__
-    >>> da_delattr('foo')
-    DelAttr delattr 'foo'
-    >>> sda_setattr = SetDelAttr().__setattr__
-    >>> sda_setattr('foo', 'bar')
-    SetDelAttr setattr 'foo' 'bar'
-    >>> sda_delattr = SetDelAttr().__delattr__
-    >>> sda_delattr('foo')
-    SetDelAttr delattr 'foo'
     >>> # If you define either set or delete, you get wrapper objects
     >>> # for both methods.  (This behavior is unchanged by #561.)
     >>> s_set = Set().__set__
@@ -119,6 +95,39 @@
     VS __index__ 0
 """
 
+cdef extern from *:
+    # type specs require a bug fix in Py3.8+ for some of these tests.
+    const int CYTHON_USE_TYPE_SPECS
+
+if not CYTHON_USE_TYPE_SPECS or sys.version_info >= (3,8):
+    __doc__ += u"""
+    >>> # If you define either setattr or delattr, you get wrapper objects
+    >>> # for both methods.  (This behavior is unchanged by #561.)
+    >>> sa_setattr = SetAttr().__setattr__
+    >>> sa_setattr('foo', 'bar')
+    SetAttr setattr 'foo' 'bar'
+    >>> sa_delattr = SetAttr().__delattr__
+    >>> sa_delattr('foo')
+    Traceback (most recent call last):
+    ...
+    AttributeError: 'special_methods_T561.SetAttr' object has no attribute 'foo'
+    >>> da_setattr = DelAttr().__setattr__
+    >>> da_setattr('foo', 'bar')
+    Traceback (most recent call last):
+    ...
+    AttributeError: 'special_methods_T561.DelAttr' object has no attribute 'foo'
+    >>> da_delattr = DelAttr().__delattr__
+    >>> da_delattr('foo')
+    DelAttr delattr 'foo'
+    >>> sda_setattr = SetDelAttr().__setattr__
+    >>> sda_setattr('foo', 'bar')
+    SetDelAttr setattr 'foo' 'bar'
+    >>> sda_delattr = SetDelAttr().__delattr__
+    >>> sda_delattr('foo')
+    SetDelAttr delattr 'foo'
+"""
+
+
 cdef class VerySpecial:
     """
     >>> vs0 = VerySpecial(0)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/ssize_t_T399.pyx cython-0.20.1+1~202203241016-9537/tests/run/ssize_t_T399.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/ssize_t_T399.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/ssize_t_T399.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 399
+# ticket: t399
 
 __doc__ = u"""
 >>> test(-2)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/starargs.pyx cython-0.20.1+1~202203241016-9537/tests/run/starargs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/starargs.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/starargs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,7 +1,6 @@
 cdef sorteditems(d):
-    l = list(d.items())
-    l.sort()
-    return tuple(l)
+    return tuple(sorted(d.items()))
+
 
 def spam(x, y, z):
     """
@@ -79,6 +78,8 @@
     >>> onlyt(1, a=2)
     Traceback (most recent call last):
     TypeError: onlyt() got an unexpected keyword argument 'a'
+    >>> test_no_copy_args(onlyt)
+    True
     """
     return a
 
@@ -114,3 +115,20 @@
     (1, ('a', 1), ('b', 2))
     """
     return a + sorteditems(k)
+
+def t_kwonly(*a, k):
+    """
+    >>> test_no_copy_args(t_kwonly, k=None)
+    True
+    """
+    return a
+
+
+def test_no_copy_args(func, **kw):
+    """
+    func is a function such that func(*args, **kw) returns args.
+    We test that no copy is made of the args tuple.
+    This tests both the caller side and the callee side.
+    """
+    args = (1, 2, 3)
+    return func(*args, **kw) is args
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/starred_target_T664.pyx cython-0.20.1+1~202203241016-9537/tests/run/starred_target_T664.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/starred_target_T664.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/starred_target_T664.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 664
+# ticket: t664
 
 def assign():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/staticmethod.pyx cython-0.20.1+1~202203241016-9537/tests/run/staticmethod.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/staticmethod.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/staticmethod.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,35 +1,17 @@
-__doc__ = u"""
->>> class1.plus1(1)
-2
->>> class2.plus1(1)
-2
->>> class3.plus1(1)
-2
->>> class4.plus1(1)
-2
->>> class4().plus1(1)
-2
->>> class4.bplus1(1)
-2
->>> class4().bplus1(1)
-2
-"""
-
 cimport cython
 
-def f_plus(a):
-    return a + 1
 
 class class1:
-    plus1 = f_plus
-
-class class2(object):
-    plus1 = f_plus
-
-cdef class class3:
-    plus1 = f_plus
-
-class class4:
+    u"""
+    >>> class1.plus1(1)
+    2
+    >>> class1().plus1(1)
+    2
+    >>> class1.bplus1(1)
+    2
+    >>> class1().bplus1(1)
+    2
+    """
     @staticmethod
     def plus1(a):
         return a + 1
@@ -49,14 +31,14 @@
     >>> obj.plus1(1)
     2
     """
-    class class5(object):
+    class class2(object):
         def __new__(cls): # implicit staticmethod
             return object.__new__(cls)
 
         @staticmethod
         def plus1(a):
             return a + 1
-    return class5
+    return class2
 
 
 cdef class BaseClass(object):
@@ -139,6 +121,14 @@
         """
         return args + tuple(sorted(kwargs.items()))
 
+    @staticmethod
+    def no_args():
+        """
+        >>> ArgsKwargs().no_args()
+        OK!
+        """
+        print("OK!")
+
 
 class StaticmethodSubclass(staticmethod):
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/static_methods.pxd cython-0.20.1+1~202203241016-9537/tests/run/static_methods.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/static_methods.pxd	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/static_methods.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,6 @@
 cdef class FromPxd:
     @staticmethod
     cdef static_cdef(int* x)
+
+    @staticmethod
+    cdef static_cdef_with_implicit_object(obj)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/static_methods.pyx cython-0.20.1+1~202203241016-9537/tests/run/static_methods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/static_methods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/static_methods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -59,6 +59,13 @@
     """
     return A.static_cdef2(&x, &y)
 
+def call_static_list_comprehension_GH1540(int x):
+    """
+    >>> call_static_list_comprehension_GH1540(5)
+    [('cdef', 5), ('cdef', 5), ('cdef', 5)]
+    """
+    return [A.static_cdef(&x) for _ in range(3)]
+
 # BROKEN
 #def call_static_cdef_untyped(a, b):
 #    """
@@ -67,6 +74,7 @@
 #    """
 #    return A.static_cdef_untyped(a, b)
 
+# UNIMPLEMENTED
 # def call_static_cpdef(int x):
 #     """
 #     >>> call_static_cpdef(2)
@@ -79,6 +87,10 @@
     cdef static_cdef(int* x):
         return 'pxd_cdef', x[0]
 
+    @staticmethod
+    cdef static_cdef_with_implicit_object(obj):
+        return obj+1
+
 def call_static_pxd_cdef(int x):
     """
     >>> call_static_pxd_cdef(2)
@@ -86,3 +98,11 @@
     """
     cdef int *x_ptr = &x
     return FromPxd.static_cdef(x_ptr)
+
+def call_static_pxd_cdef_with_implicit_object(int x):
+    """
+    # https://github.com/cython/cython/issues/3174
+    >>> call_static_pxd_cdef_with_implicit_object(2)
+    3
+    """
+    return FromPxd.static_cdef_with_implicit_object(x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/str_char_coercion_T412.pyx cython-0.20.1+1~202203241016-9537/tests/run/str_char_coercion_T412.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/str_char_coercion_T412.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/str_char_coercion_T412.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 412
+# ticket: t412
 
 cdef int   i = 'x'
 cdef char  c = 'x'
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/str_default_auto_encoding.pyx cython-0.20.1+1~202203241016-9537/tests/run/str_default_auto_encoding.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/str_default_auto_encoding.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/str_default_auto_encoding.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,8 @@
-# cython: c_string_type = str
-# cython: c_string_encoding = default
+# cython: c_string_encoding=default
+# cython: c_string_type=str
+
+# NOTE: the directive order above is specifically meant to trigger (and confuse) the
+# source encoding detector with "coding=default".
 
 import sys
 if sys.version_info[0] >= 3:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/strfunction.pyx cython-0.20.1+1~202203241016-9537/tests/run/strfunction.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/strfunction.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/strfunction.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -5,6 +5,8 @@
    'test'
 """
 
+cimport cython
+
 s = str
 z = str('test')
 
@@ -39,3 +41,33 @@
 
 #def csub(string):
 #    return csubs(string)
+
+
+@cython.test_fail_if_path_exists("//SimpleCallNode")
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+def typed(str s):
+    """
+    >>> print(typed(None))
+    None
+    >>> type(typed(None)) is type(typed(None))
+    True
+    >>> print(typed('abc'))
+    abc
+    >>> type(typed('abc')) is type(typed('abc'))
+    True
+    """
+    return str(s)
+
+
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode",
+    "//PythonCapiCallNode",
+)
+def typed_not_none(str s not None):
+    """
+    >>> print(typed('abc'))
+    abc
+    >>> type(typed('abc')) is type(typed('abc'))
+    True
+    """
+    return str(s)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/strmethods.pyx cython-0.20.1+1~202203241016-9537/tests/run/strmethods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/strmethods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/strmethods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -55,6 +55,26 @@
       return s.endswith(sub, start, stop)
 
 
+def object_as_name(object):
+    """
+    >>> object_as_name('abx')
+    True
+    >>> object_as_name('abc')
+    False
+    """
+    return object.endswith("x")
+
+
+def str_as_name(str):
+    """
+    >>> str_as_name('abx')
+    True
+    >>> str_as_name('abc')
+    False
+    """
+    return str.endswith("x")
+
+
 @cython.test_assert_path_exists(
     "//SimpleCallNode",
     "//SimpleCallNode//NoneCheckNode",
@@ -96,9 +116,9 @@
     True
     >>> mod_format(format2, ('XYZ', 'ABC')) == 'abcXYZdefABCghi'  or  mod_format(format2, ('XYZ', 'ABC'))
     True
-    >>> mod_format(None, 'sa')
+    >>> mod_format(None, 'sa')  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: unsupported operand type(s) for %: 'NoneType' and 'str'
+    TypeError: ...NoneType...
     >>> class RMod(object):
     ...     def __rmod__(self, other):
     ...         return 123
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/str_subclass_kwargs.pyx cython-0.20.1+1~202203241016-9537/tests/run/str_subclass_kwargs.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/str_subclass_kwargs.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/str_subclass_kwargs.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,21 @@
+def test_str_subclass_kwargs(k=None):
+    """
+    Test passing keywords with names that are not of type ``str``
+    but a subclass:
+
+    >>> class StrSubclass(str):
+    ...     pass
+    >>> class StrNoCompare(str):
+    ...     def __eq__(self, other):
+    ...         raise RuntimeError("do not compare me")
+    ...     def __hash__(self):
+    ...         return hash(str(self))
+    >>> kwargs = {StrSubclass('k'): 'value'}
+    >>> test_str_subclass_kwargs(**kwargs)
+    'value'
+    >>> kwargs = {StrNoCompare('k'): 'value'}
+    >>> test_str_subclass_kwargs(**kwargs)
+    Traceback (most recent call last):
+    RuntimeError: do not compare me
+    """
+    return k
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/struct_conversion.pyx cython-0.20.1+1~202203241016-9537/tests/run/struct_conversion.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/struct_conversion.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/struct_conversion.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -28,10 +28,9 @@
     """
     >>> sorted(test_constructor_kwds(1.25, 2.5, 128).items())
     [('color', 128), ('x', 1.25), ('y', 2.5)]
-    >>> test_constructor_kwds(1.25, 2.5, None)
+    >>> test_constructor_kwds(1.25, 2.5, None)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ...
-    TypeError: an integer is required
+    TypeError:... int...
     """
     cdef Point p = Point(x=x, y=y, color=color)
     return p
@@ -41,10 +40,9 @@
     """
     >>> sorted(return_constructor_kwds(1.25, 2.5, 128).items())
     [('color', 128), ('x', 1.25), ('y', 2.5)]
-    >>> return_constructor_kwds(1.25, 2.5, None)
+    >>> return_constructor_kwds(1.25, 2.5, None)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    ...
-    TypeError: an integer is required
+    TypeError:... int...
     """
     return Point(x=x, y=y, color=color)
 
@@ -169,3 +167,19 @@
                                             nested.mystruct.s.decode('UTF-8'),
                                             nested.d)
 
+cdef struct OverriddenCname:
+    int x "not_x"
+
+def test_obj_to_struct_cnames(OverriddenCname s):
+    """
+    >>> test_obj_to_struct_cnames({ 'x': 1 })
+    1
+    """
+    print(s.x)
+
+def test_struct_to_obj_cnames():
+    """
+    >>> test_struct_to_obj_cnames()
+    {'x': 2}
+    """
+    return OverriddenCname(2)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/subop.pyx cython-0.20.1+1~202203241016-9537/tests/run/subop.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/subop.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/subop.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -191,3 +191,27 @@
     ... except TypeError: pass
     """
     return 2**30 - x
+
+
+def sub0(x):
+    """
+    >>> sub0(0)
+    (0, 0)
+    >>> sub0(1)
+    (1, -1)
+    >>> sub0(-1)
+    (-1, 1)
+    >>> sub0(99)
+    (99, -99)
+    >>> a, b = sub0(2**32)
+    >>> bigint(a)
+    4294967296
+    >>> bigint(b)
+    -4294967296
+    >>> a, b = sub0(-2**32)
+    >>> bigint(a)
+    -4294967296
+    >>> bigint(b)
+    4294967296
+    """
+    return x - 0, 0 - x
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/switch.pyx cython-0.20.1+1~202203241016-9537/tests/run/switch.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/switch.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/switch.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,5 @@
 # mode: run
+# cython: linetrace=True
 
 cimport cython
 
@@ -143,6 +144,75 @@
     return -1
 
 
+
+@cython.test_assert_path_exists(
+    '//SwitchStatNode',
+    '//SwitchStatNode//SwitchStatNode',
+)
+@cython.test_fail_if_path_exists('//BoolBinopNode', '//PrimaryCmpNode')
+def switch_in_switch(int x, int y):
+    """
+    >>> switch_in_switch(1, 1)
+    (1, 1)
+    >>> switch_in_switch(1, 2)
+    (1, 2)
+    >>> switch_in_switch(1, 4)
+    (1, 3)
+
+    >>> switch_in_switch(2, 1)
+    (2, 1)
+    >>> switch_in_switch(2, 2)
+    (2, 2)
+    >>> switch_in_switch(2, 3)
+    (2, 3)
+    >>> switch_in_switch(2, 4)
+    (2, 4)
+    >>> switch_in_switch(2, 20)
+    (2, 4)
+
+    >>> switch_in_switch(3, 0)
+    False
+    >>> switch_in_switch(3, 1)
+    True
+    >>> switch_in_switch(3, 2)
+    True
+    >>> switch_in_switch(3, 3)
+    True
+    >>> switch_in_switch(3, 4)
+    False
+
+    >>> switch_in_switch(20, 0)
+    True
+    >>> switch_in_switch(20, 1)
+    False
+    >>> switch_in_switch(20, 3)
+    False
+    >>> switch_in_switch(20, 4)
+    True
+    """
+    if x == 1:
+        if y == 1:
+            return 1,1
+        elif y == 2:
+            return 1,2
+        else:
+            return 1,3
+    elif x == 2:
+        if y == 1:
+            return 2,1
+        elif y == 2:
+            return 2,2
+        elif y == 3:
+            return 2,3
+        else:
+            return 2,4
+    elif x == 3:
+        return y in (1,2,3)
+    else:
+        return y not in (1,2,3)
+    return 'FAILED'
+
+
 @cython.test_assert_path_exists('//SwitchStatNode')
 @cython.test_fail_if_path_exists('//IfStatNode')
 def switch_or(int x):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/switch_transform.pyx cython-0.20.1+1~202203241016-9537/tests/run/switch_transform.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/switch_transform.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/switch_transform.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,5 @@
 # cython: optimize.use_switch=False
+# cython: linetrace=True
 
 cdef extern from *:
     enum:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/temp_alloc_T409.pyx cython-0.20.1+1~202203241016-9537/tests/run/temp_alloc_T409.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/temp_alloc_T409.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/temp_alloc_T409.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 409
+# ticket: t409
 # Extracted from sage/plot/plot3d/index_face_set.pyx:502
 # Turns out to be a bug in implementation of PEP 3132 (Extended Iterable Unpacking)
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/temp_sideeffects_T654.pyx cython-0.20.1+1~202203241016-9537/tests/run/temp_sideeffects_T654.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/temp_sideeffects_T654.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/temp_sideeffects_T654.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 654
+# ticket: t654
 
 # function call arguments
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_asyncgen.py cython-0.20.1+1~202203241016-9537/tests/run/test_asyncgen.py
--- cython-0.20.1+1~201611251650-6686/tests/run/test_asyncgen.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_asyncgen.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,1373 @@
+# cython: language_level=3, binding=True
+# mode: run
+# tag: pep525, asyncfor, await
+
+from __future__ import generator_stop
+
+import os
+import sys
+#import inspect
+#import types
+import unittest
+import contextlib
+
+#from unittest import mock
+
+#from test.support import import_module
+
+#asyncio = import_module("asyncio")
+
+ZERO = 0
+
+try:
+    if sys.version_info[:2] == (3, 4):
+        # asnycio in Py3.4 does not support awaitable coroutines (requires iterators instead)
+        raise ImportError
+    import asyncio
+except ImportError:
+    try:
+        from unittest import skip
+    except ImportError:
+        def requires_asyncio(c):
+            return None
+    else:
+        requires_asyncio = skip("tests require asyncio")
+    asyncio = None
+else:
+    def requires_asyncio(c):
+        return c
+
+
+def needs_py36_asyncio(f):
+    if sys.version_info >= (3, 6) or asyncio is None:
+        # Py<3.4 doesn't have asyncio at all => avoid having to special case 2.6's unittest below
+        return f
+
+    from unittest import skip
+    return skip("needs Python 3.6 or later")(f)
+
+
+try:
+    from types import coroutine as types_coroutine
+except ImportError:
+    def types_coroutine(func):
+        from functools import wraps
+        wrapped = wraps(func)
+
+        # copied from types.py in Py3.6
+        class _GeneratorWrapper(object):
+            # TODO: Implement this in C.
+            def __init__(self, gen):
+                self.__wrapped = gen
+                self.__isgen = hasattr(gen, 'gi_running')
+                self.__name__ = getattr(gen, '__name__', None)
+                self.__qualname__ = getattr(gen, '__qualname__', None)
+
+            def send(self, val):
+                return self.__wrapped.send(val)
+
+            def throw(self, tp, *rest):
+                return self.__wrapped.throw(tp, *rest)
+
+            def close(self):
+                return self.__wrapped.close()
+
+            @property
+            def gi_code(self):
+                return self.__wrapped.gi_code
+
+            @property
+            def gi_frame(self):
+                return self.__wrapped.gi_frame
+
+            @property
+            def gi_running(self):
+                return self.__wrapped.gi_running
+
+            @property
+            def gi_yieldfrom(self):
+                return self.__wrapped.gi_yieldfrom
+
+            cr_code = gi_code
+            cr_frame = gi_frame
+            cr_running = gi_running
+            cr_await = gi_yieldfrom
+
+            def __next__(self):
+                return next(self.__wrapped)
+            next = __next__
+
+            def __iter__(self):
+                if self.__isgen:
+                    return self.__wrapped
+                return self
+
+            __await__ = __iter__
+
+        @wrapped
+        def call(*args, **kwargs):
+            return wrapped(_GeneratorWrapper(func(*args, **kwargs)))
+
+        return call
+
+try:
+    from inspect import isawaitable as inspect_isawaitable
+except ImportError:
+    def inspect_isawaitable(o):
+        return hasattr(o, '__await__')
+
+
+# compiled exec()
+def exec(code_string, l, g):
+    from Cython.Compiler.Errors import CompileError
+    from Cython.Shadow import inline
+    try:
+        from StringIO import StringIO
+    except ImportError:
+        from io import StringIO
+
+    old_stderr = sys.stderr
+    try:
+        sys.stderr = StringIO()
+        ns = inline(code_string, locals=l, globals=g, lib_dir=os.path.dirname(__file__))
+    except CompileError as exc:
+        raise SyntaxError(str(exc))
+    finally:
+        sys.stderr = old_stderr
+    g.update(ns)
+
+
+class AwaitException(Exception):
+    pass
+
+
+@types_coroutine
+def awaitable(*, throw=False):
+    if throw:
+        yield ('throw',)
+    else:
+        yield ('result',)
+
+
+def run_until_complete(coro):
+    exc = False
+    while True:
+        try:
+            if exc:
+                exc = False
+                fut = coro.throw(AwaitException)
+            else:
+                fut = coro.send(None)
+        except StopIteration as ex:
+            return ex.args[0]
+
+        if fut == ('throw',):
+            exc = True
+
+
+def to_list(gen):
+    async def iterate():
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    return run_until_complete(iterate())
+
+
+class AsyncGenSyntaxTest(unittest.TestCase):
+
+    @contextlib.contextmanager
+    def assertRaisesRegex(self, exc_type, regex):
+        # the error messages usually don't match, so we just ignore them
+        try:
+            yield
+        except exc_type:
+            self.assertTrue(True)
+        else:
+            self.assertTrue(False)
+
+    def test_async_gen_syntax_01(self):
+        code = '''async def foo():
+            await abc
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_02(self):
+        code = '''async def foo():
+            yield from 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'yield from.*inside async'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_03(self):
+        code = '''async def foo():
+            await abc
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_04(self):
+        code = '''async def foo():
+            yield
+            return 123
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+    def test_async_gen_syntax_05(self):
+        code = '''async def foo():
+            if 0:
+                yield
+            return 12
+        '''
+
+        with self.assertRaisesRegex(SyntaxError, 'return.*value.*async gen'):
+            exec(code, {}, {})
+
+
+class AsyncGenTest(unittest.TestCase):
+
+    if sys.version_info < (3, 3):
+        @contextlib.contextmanager
+        def assertRaisesRegex(self, exc_type, regex=None):
+            # the error messages usually don't match, so we just ignore them
+            try:
+                yield
+            except exc_type:
+                self.assertTrue(True)
+            else:
+                self.assertTrue(False)
+
+    def compare_generators(self, sync_gen, async_gen):
+        def sync_iterate(g):
+            res = []
+            while True:
+                try:
+                    res.append(next(g))
+                except StopIteration:
+                    res.append('STOP')
+                    break
+                except Exception as ex:
+                    res.append(str(type(ex)))
+            return res
+
+        def async_iterate(g):
+            res = []
+            while True:
+                an = g.__anext__()
+                try:
+                    while True:
+                        try:
+                            next(an)
+                        except StopIteration as ex:
+                            if ex.args:
+                                res.append(ex.args[0])
+                                break
+                            else:
+                                res.append('EMPTY StopIteration')
+                                break
+                        except StopAsyncIteration:
+                            raise
+                        except Exception as ex:
+                            res.append(str(type(ex)))
+                            break
+                except StopAsyncIteration:
+                    res.append('STOP')
+                    break
+            return res
+
+        sync_gen_result = sync_iterate(sync_gen)
+        async_gen_result = async_iterate(async_gen)
+        self.assertEqual(sync_gen_result, async_gen_result)
+        return async_gen_result
+
+    def test_async_gen_iteration_01(self):
+        async def gen():
+            await awaitable()
+            a = yield 123
+            self.assertIs(a, None)
+            await awaitable()
+            yield 456
+            await awaitable()
+            yield 789
+
+        self.assertEqual(to_list(gen()), [123, 456, 789])
+
+    def test_async_gen_iteration_02(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable()
+
+        g = gen()
+        ai = g.__aiter__()
+
+        an = ai.__anext__()
+        self.assertEqual(next(an), ('result',))
+
+        try:
+            next(an)
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        an = ai.__anext__()
+        self.assertEqual(next(an), ('result',))
+
+        try:
+            next(an)
+        except StopAsyncIteration as ex:
+            self.assertFalse(ex.args)
+        else:
+            self.fail('StopAsyncIteration was not raised')
+
+    def test_async_gen_exception_03(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            await awaitable(throw=True)
+            yield 456
+
+        with self.assertRaises(AwaitException):
+            to_list(gen())
+
+    def test_async_gen_exception_04(self):
+        async def gen():
+            await awaitable()
+            yield 123
+            1 / ZERO
+
+        g = gen()
+        ai = g.__aiter__()
+
+        an = ai.__anext__()
+        self.assertEqual(next(an), ('result',))
+
+        try:
+            next(an)
+        except StopIteration as ex:
+            self.assertEqual(ex.args[0], 123)
+        else:
+            self.fail('StopIteration was not raised')
+
+        with self.assertRaises(ZeroDivisionError):
+            next(ai.__anext__())
+
+    def test_async_gen_exception_05(self):
+        async def gen():
+            yield 123
+            raise StopAsyncIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopAsyncIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_06(self):
+        async def gen():
+            yield 123
+            raise StopIteration
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'async generator.*StopIteration'):
+            to_list(gen())
+
+    def test_async_gen_exception_07(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / ZERO
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                1 / ZERO
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_08(self):
+        def sync_gen():
+            try:
+                yield 1
+            finally:
+                yield 2
+                1 / ZERO
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                yield 1
+                await awaitable()
+            finally:
+                await awaitable()
+                yield 2
+                1 / ZERO
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_09(self):
+        def sync_gen():
+            try:
+                yield 1
+                1 / ZERO
+            finally:
+                yield 2
+                yield 3
+
+            yield 100
+
+        async def async_gen():
+            try:
+                await awaitable()
+                yield 1
+                1 / ZERO
+            finally:
+                yield 2
+                await awaitable()
+                yield 3
+
+            yield 100
+
+        self.compare_generators(sync_gen(), async_gen())
+
+    def test_async_gen_exception_10(self):
+        async def gen():
+            yield 123
+        with self.assertRaisesRegex(TypeError,
+                                    "non-None value .* async generator"):
+            gen().__anext__().send(100)
+
+    def test_async_gen_exception_11(self):
+        def sync_gen():
+            yield 10
+            yield 20
+
+        def sync_gen_wrapper():
+            yield 1
+            sg = sync_gen()
+            sg.send(None)
+            try:
+                sg.throw(GeneratorExit())
+            except GeneratorExit:
+                yield 2
+            yield 3
+
+        async def async_gen():
+            yield 10
+            yield 20
+
+        async def async_gen_wrapper():
+            yield 1
+            asg = async_gen()
+            await asg.asend(None)
+            try:
+                await asg.athrow(GeneratorExit())
+            except GeneratorExit:
+                yield 2
+            yield 3
+
+        self.compare_generators(sync_gen_wrapper(), async_gen_wrapper())
+
+    def test_async_gen_api_01(self):
+        async def gen():
+            yield 123
+
+        g = gen()
+
+        self.assertEqual(g.__name__, 'gen')
+        g.__name__ = '123' if sys.version_info[0] >= 3 else b'123'
+        self.assertEqual(g.__name__, '123')
+
+        self.assertIn('.gen', g.__qualname__)
+        g.__qualname__ = '123' if sys.version_info[0] >= 3 else b'123'
+        self.assertEqual(g.__qualname__, '123')
+
+        #self.assertIsNone(g.ag_await)
+        #self.assertIsInstance(g.ag_frame, types.FrameType)
+        self.assertFalse(g.ag_running)
+        #self.assertIsInstance(g.ag_code, types.CodeType)
+
+        self.assertTrue(inspect_isawaitable(g.aclose()))
+
+
+@requires_asyncio
+class AsyncGenAsyncioTest(unittest.TestCase):
+
+    def setUp(self):
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(None)
+
+    def tearDown(self):
+        self.loop.close()
+        self.loop = None
+
+    async def to_list(self, gen):
+        res = []
+        async for i in gen:
+            res.append(i)
+        return res
+
+    def test_async_gen_asyncio_01(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01)
+            yield 2
+            await asyncio.sleep(0.01)
+            return
+            yield 3
+
+        res = self.loop.run_until_complete(self.to_list(gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_02(self):
+        async def gen():
+            yield 1
+            await asyncio.sleep(0.01)
+            yield 2
+            1 / ZERO
+            yield 3
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(self.to_list(gen()))
+
+    def test_async_gen_asyncio_03(self):
+        loop = self.loop
+
+        class Gen:
+            async def __aiter__(self):
+                yield 1
+                await asyncio.sleep(0.01)
+                yield 2
+
+        res = loop.run_until_complete(self.to_list(Gen()))
+        self.assertEqual(res, [1, 2])
+
+    def test_async_gen_asyncio_anext_04(self):
+        async def foo():
+            yield 1
+            await asyncio.sleep(0.01)
+            try:
+                yield 2
+                yield 3
+            except ZeroDivisionError:
+                yield 1000
+            await asyncio.sleep(0.01)
+            yield 4
+
+        async def run1():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            self.assertEqual(await it.__anext__(), 3)
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        async def run2():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), 1)
+            self.assertEqual(await it.__anext__(), 2)
+            try:
+                it.__anext__().throw(ZeroDivisionError)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1000)
+            else:
+                self.fail('StopIteration was not raised')
+            self.assertEqual(await it.__anext__(), 4)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run1())
+        self.loop.run_until_complete(run2())
+
+    def test_async_gen_asyncio_anext_05(self):
+        async def foo():
+            v = yield 1
+            v = yield v
+            yield v * 100
+
+        async def run():
+            it = foo().__aiter__()
+
+            try:
+                it.__anext__().send(None)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(10)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 10)
+            else:
+                self.fail('StopIteration was not raised')
+
+            try:
+                it.__anext__().send(12)
+            except StopIteration as ex:
+                self.assertEqual(ex.args[0], 1200)
+            else:
+                self.fail('StopIteration was not raised')
+
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_anext_06(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        with self.assertRaises(StopIteration):
+            g.send(None)
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_anext_tuple(self):
+        async def foo():
+            try:
+                yield (1,)
+            except ZeroDivisionError:
+                yield (2,)
+
+        async def run():
+            it = foo().__aiter__()
+
+            self.assertEqual(await it.__anext__(), (1,))
+            with self.assertRaises(StopIteration) as cm:
+                it.__anext__().throw(ZeroDivisionError)
+            self.assertEqual(cm.exception.args[0], (2,))
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_anext_stopiteration(self):
+        async def foo():
+            try:
+                yield StopIteration(1)
+            except ZeroDivisionError:
+                yield StopIteration(3)
+
+        async def run():
+            it = foo().__aiter__()
+
+            v = await it.__anext__()
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 1)
+            with self.assertRaises(StopIteration) as cm:
+                it.__anext__().throw(ZeroDivisionError)
+            v = cm.exception.args[0]
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 3)
+            with self.assertRaises(StopAsyncIteration):
+                await it.__anext__()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_06(self):
+        async def foo():
+            try:
+                yield 1
+                1 / ZERO
+            finally:
+                await asyncio.sleep(0.01)
+                yield 12
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        with self.assertRaisesRegex(
+                RuntimeError,
+                "async generator ignored GeneratorExit"):
+            self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_aclose_07(self):
+        DONE = 0
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                1 / ZERO
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_aclose_08(self):
+        DONE = 0
+
+        fut = asyncio.Future(loop=self.loop)
+
+        async def foo():
+            nonlocal DONE
+            try:
+                yield 1
+                await fut
+                DONE += 1000
+                yield 2
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE += 1
+            DONE += 1000
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            self.assertEqual(await it.__anext__(), 1)
+            await gen.aclose()
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+        # Silence ResourceWarnings
+        fut.cancel()
+        self.loop.run_until_complete(asyncio.sleep(0.01))
+
+    @needs_py36_asyncio
+    def test_async_gen_asyncio_gc_aclose_09(self):
+        DONE = 0
+
+        async def gen():
+            nonlocal DONE
+            try:
+                while True:
+                    yield 1
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE = 1
+
+        async def run():
+            g = gen()
+            await g.__anext__()
+            await g.__anext__()
+            del g
+
+            await asyncio.sleep(0.2)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_aclose_10(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        g.close()
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            await g.aclose()
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_aclose_11(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+            yield
+        g = foo()
+        g.send(None)
+        with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'):
+            g.close()
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            yield
+            DONE += 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaisesRegex(RuntimeError, 'ignored GeneratorExit'):
+                await g.aclose()
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 10)
+
+    def test_async_gen_asyncio_aclose_12(self):
+        DONE = 0
+
+        async def target():
+            await asyncio.sleep(0.01)
+            1 / ZERO
+
+        async def foo():
+            nonlocal DONE
+            task = self.loop.create_task(target())
+            try:
+                yield 1
+            finally:
+                try:
+                    await task
+                except ZeroDivisionError:
+                    DONE = 1
+
+        async def run():
+            gen = foo()
+            it = gen.__aiter__()
+            await it.__anext__()
+            await gen.aclose()
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_01(self):
+        DONE = 0
+
+        # Sanity check:
+        def sgen():
+            v = yield 1
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.send(100)
+        self.assertEqual(v, 200)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01)
+                v = yield 1
+                await asyncio.sleep(0.01)
+                yield v * 2
+                await asyncio.sleep(0.01)
+                return
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.asend(100)
+            self.assertEqual(v, 200)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_02(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            await asyncio.sleep(delay)
+            1 / ZERO
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                assert sys.exc_info()[0] == ZeroDivisionError
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE += 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(ZeroDivisionError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_asend_03(self):
+        DONE = 0
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01)
+                v = yield 1
+                await sleep_n_crash(0.01)
+                DONE += 1000
+                yield v * 2
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            await g.asend(100)
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_01(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        # Sanity check:
+        def sgen():
+            try:
+                v = yield 1
+            except FooEr:
+                v = 1000
+            yield v * 2
+        sg = sgen()
+        v = sg.send(None)
+        self.assertEqual(v, 1)
+        v = sg.throw(FooEr)
+        self.assertEqual(v, 2000)
+        with self.assertRaises(StopIteration):
+            sg.send(None)
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01)
+                try:
+                    v = yield 1
+                except FooEr:
+                    v = 1000
+                    await asyncio.sleep(0.01)
+                yield v * 2
+                await asyncio.sleep(0.01)
+                # return
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            v = await g.athrow(FooEr)
+            self.assertEqual(v, 2000)
+
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_02(self):
+        DONE = 0
+
+        class FooEr(Exception):
+            pass
+
+        async def sleep_n_crash(delay):
+            fut = asyncio.ensure_future(asyncio.sleep(delay),
+                                        loop=self.loop)
+            self.loop.call_later(delay / 2, lambda: fut.cancel())
+            return await fut
+
+        async def gen():
+            nonlocal DONE
+            try:
+                await asyncio.sleep(0.01)
+                try:
+                    v = yield 1
+                except FooEr:
+                    await sleep_n_crash(0.01)
+                yield v * 2
+                await asyncio.sleep(0.01)
+                # return
+            finally:
+                await asyncio.sleep(0.01)
+                await asyncio.sleep(0.01)
+                DONE = 1
+
+        async def run():
+            g = gen()
+
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+
+            try:
+                await g.athrow(FooEr)
+            except asyncio.CancelledError:
+                self.assertEqual(DONE, 1)
+                raise
+            else:
+                self.fail('CancelledError was not raised')
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 1)
+
+    def test_async_gen_asyncio_athrow_03(self):
+        DONE = 0
+
+        # test synchronous generators
+        def foo():
+            try:
+                yield
+            except:
+                pass
+        g = foo()
+        g.send(None)
+        with self.assertRaises(StopIteration):
+            g.throw(ValueError)
+
+        # now with asynchronous generators
+
+        async def gen():
+            nonlocal DONE
+            try:
+                yield
+            except:
+                pass
+            DONE = 1
+
+        async def run():
+            nonlocal DONE
+            g = gen()
+            await g.asend(None)
+            with self.assertRaises(StopAsyncIteration):
+                await g.athrow(ValueError)
+            DONE += 10
+
+        self.loop.run_until_complete(run())
+        self.assertEqual(DONE, 11)
+
+    def test_async_gen_asyncio_athrow_tuple(self):
+        async def gen():
+            try:
+                yield 1
+            except ZeroDivisionError:
+                yield (2,)
+
+        async def run():
+            g = gen()
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+            v = await g.athrow(ZeroDivisionError)
+            self.assertEqual(v, (2,))
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_asyncio_athrow_stopiteration(self):
+        async def gen():
+            try:
+                yield 1
+            except ZeroDivisionError:
+                yield StopIteration(2)
+
+        async def run():
+            g = gen()
+            v = await g.asend(None)
+            self.assertEqual(v, 1)
+            v = await g.athrow(ZeroDivisionError)
+            self.assertIsInstance(v, StopIteration)
+            self.assertEqual(v.value, 2)
+            with self.assertRaises(StopAsyncIteration):
+                await g.asend(None)
+
+        self.loop.run_until_complete(run())
+
+    @needs_py36_asyncio
+    def test_async_gen_asyncio_shutdown_01(self):
+        finalized = 0
+
+        async def waiter(timeout):
+            nonlocal finalized
+            try:
+                await asyncio.sleep(timeout)
+                yield 1
+            finally:
+                await asyncio.sleep(0)
+                finalized += 1
+
+        async def wait():
+            async for _ in waiter(1):
+                pass
+
+        t1 = self.loop.create_task(wait())
+        t2 = self.loop.create_task(wait())
+
+        self.loop.run_until_complete(asyncio.sleep(0.1))
+
+
+        # Silence warnings
+        t1.cancel()
+        t2.cancel()
+
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t1)
+        with self.assertRaises(asyncio.CancelledError):
+            self.loop.run_until_complete(t2)
+
+        self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+
+        self.assertEqual(finalized, 2)
+
+    """
+    def test_async_gen_expression_01(self):
+        async def arange(n):
+            for i in range(n):
+                await asyncio.sleep(0.01)
+                yield i
+
+        def make_arange(n):
+            # This syntax is legal starting with Python 3.7
+            return (i * 2 async for i in arange(n))
+
+        async def run():
+            return [i async for i in make_arange(10)]
+
+        res = self.loop.run_until_complete(run())
+        self.assertEqual(res, [i * 2 for i in range(10)])
+
+    def test_async_gen_expression_02(self):
+        async def wrap(n):
+            await asyncio.sleep(0.01)
+            return n
+
+        def make_arange(n):
+            # This syntax is legal starting with Python 3.7
+            return (i * 2 for i in range(n) if await wrap(i))
+
+        async def run():
+            return [i async for i in make_arange(10)]
+
+        res = self.loop.run_until_complete(run())
+        self.assertEqual(res, [i * 2 for i in range(1, 10)])
+    """
+
+    def test_asyncgen_nonstarted_hooks_are_cancellable(self):
+        # See https://bugs.python.org/issue38013
+        messages = []
+
+        def exception_handler(loop, context):
+            messages.append(context)
+
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def main():
+            # loop = asyncio.get_running_loop()
+            loop = self.loop
+            loop.set_exception_handler(exception_handler)
+
+            async for i in async_iterate():
+                break
+
+        # asyncio.run(main())
+        self.loop.run_until_complete(main())
+
+        self.assertEqual([], messages)
+
+    def test_async_gen_await_same_anext_coro_twice(self):
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def run():
+            it = async_iterate()
+            nxt = it.__anext__()
+            await nxt
+            with self.assertRaisesRegex(
+                    RuntimeError,
+                    r"cannot reuse already awaited __anext__\(\)/asend\(\)"
+            ):
+                await nxt
+
+            await it.aclose()  # prevent unfinished iterator warning
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_await_same_aclose_coro_twice(self):
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def run():
+            it = async_iterate()
+            nxt = it.aclose()
+            await nxt
+            with self.assertRaisesRegex(
+                    RuntimeError,
+                    r"cannot reuse already awaited aclose\(\)/athrow\(\)"
+            ):
+                await nxt
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_aclose_twice_with_different_coros(self):
+        # Regression test for https://bugs.python.org/issue39606
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def run():
+            it = async_iterate()
+            await it.aclose()
+            await it.aclose()
+
+        self.loop.run_until_complete(run())
+
+    def test_async_gen_aclose_after_exhaustion(self):
+        # Regression test for https://bugs.python.org/issue39606
+        async def async_iterate():
+            yield 1
+            yield 2
+
+        async def run():
+            it = async_iterate()
+            async for _ in it:
+                pass
+            await it.aclose()
+
+        self.loop.run_until_complete(run())
+
+    """
+    def test_async_gen_aclose_compatible_with_get_stack(self):
+        async def async_generator():
+            yield object()
+
+        async def run():
+            ag = async_generator()
+            self.loop.create_task(ag.aclose())
+            tasks = asyncio.all_tasks()
+            for task in tasks:
+                # No AttributeError raised
+                task.get_stack()
+
+        self.loop.run_until_complete(run())
+    """
+
+
+if __name__ == "__main__":
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_coroutines_pep492.pyx cython-0.20.1+1~202203241016-9537/tests/run/test_coroutines_pep492.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/test_coroutines_pep492.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_coroutines_pep492.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,11 @@
 # cython: language_level=3, binding=True
 # mode: run
-# tag: pep492, asyncfor, await
+# tag: pep492, pep530, asyncfor, await
+
+###########
+# This file is a copy of the corresponding test file in CPython.
+# Please keep in sync and do not add non-upstream tests.
+###########
 
 import re
 import gc
@@ -20,7 +25,7 @@
 try:
     from types import coroutine as types_coroutine
 except ImportError:
-    # duck typed types.coroutine() decorator copied from types.py in Py3.5
+    # duck typed types_coroutine() decorator copied from types.py in Py3.5
     class types_coroutine(object):
         def __init__(self, gen):
             self._gen = gen
@@ -54,6 +59,22 @@
         def __call__(self, *args, **kwargs):
             return self._GeneratorWrapper(self._gen(*args, **kwargs))
 
+try:
+    from sys import getrefcount
+except ImportError:
+    from cpython.ref cimport PyObject
+    def getrefcount(obj):
+        gc.collect()
+        # PyPy needs to execute a bytecode to run the finalizers
+        exec('', {}, {})
+        return (obj).ob_refcnt
+
+
+def no_pypy(f):
+    import platform
+    if platform.python_implementation() == 'PyPy':
+        return unittest.skip("excluded in PyPy")
+
 
 # compiled exec()
 def exec(code_string, l, g):
@@ -66,12 +87,16 @@
     old_stderr = sys.stderr
     try:
         sys.stderr = StringIO()
-        ns = inline(code_string, locals=l, globals=g, lib_dir=os.path.dirname(__file__))
+        ns = inline(code_string, locals=l, globals=g, lib_dir=os.path.dirname(__file__), language_level=3)
     finally:
         sys.stderr = old_stderr
     g.update(ns)
 
 
+def compile(code_string, module, level):
+    exec(code_string, {}, {})
+
+
 class AsyncYieldFrom(object):
     def __init__(self, obj):
         self.obj = obj
@@ -90,7 +115,7 @@
 
 def run_async(coro):
     #assert coro.__class__ is types.GeneratorType
-    assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__
+    assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__
 
     buffer = []
     result = None
@@ -104,7 +129,7 @@
 
 
 def run_async__await__(coro):
-    assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__
+    assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__
     aw = coro.__await__()
     buffer = []
     result = None
@@ -130,38 +155,375 @@
         gc.collect()
 
 
-def min_py27(method):
-    return None if sys.version_info < (2, 7) else method
-
+@contextlib.contextmanager
+def captured_stderr():
+    try:
+        # StringIO.StringIO() also accepts str in Py2, io.StringIO() does not
+        from StringIO import StringIO
+    except ImportError:
+        from io import StringIO
 
-def ignore_py26(manager):
-    @contextlib.contextmanager
-    def dummy():
-        yield
-    return dummy() if sys.version_info < (2, 7) else manager
+    orig_stderr = sys.stderr
+    try:
+        sys.stderr = StringIO()
+        yield sys.stderr
+    finally:
+        sys.stderr = orig_stderr
 
 
 class AsyncBadSyntaxTest(unittest.TestCase):
 
     @contextlib.contextmanager
     def assertRaisesRegex(self, exc_type, regex):
+        class Holder(object):
+            exception = None
+
+        holder = Holder()
         # the error messages usually don't match, so we just ignore them
         try:
-            yield
-        except exc_type:
+            yield holder
+        except exc_type as exc:
+            holder.exception = exc
             self.assertTrue(True)
         else:
             self.assertTrue(False)
 
-    def test_badsyntax_9(self):
-        ns = {}
-        for comp in {'(await a for a in b)',
-                     '[await a for a in b]',
-                     '{await a for a in b}',
-                     '{await a: a for a in b}'}:
+    def test_badsyntax_1(self):
+        samples = [
+            """def foo():
+                await something()
+            """,
+
+            """await something()""",
+
+            """async def foo():
+                yield from []
+            """,
+
+            """async def foo():
+                await await fut
+            """,
+
+            """async def foo(a=await something()):
+                pass
+            """,
+
+            #"""async def foo(a:await something()):
+            #    pass
+            #""", # No longer an error with pep-563 (although still nonsense)
+            # Some other similar tests have also been commented out
+
+            """async def foo():
+                def bar():
+                 [i async for i in els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [await i for i in els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els
+                    async for b in els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els
+                    for c in b
+                    async for b in els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els
+                    async for b in els
+                    for c in b]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els
+                    for b in await els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els
+                    for b in els
+                        if await b]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in await els]
+            """,
+
+            """async def foo():
+                def bar():
+                 [i for i in els if await i]
+            """,
+
+            """def bar():
+                 [i async for i in els]
+            """,
+
+            """def bar():
+                 [await i for i in els]
+            """,
+
+            """def bar():
+                 [i for i in els
+                    async for b in els]
+            """,
+
+            """def bar():
+                 [i for i in els
+                    for c in b
+                    async for b in els]
+            """,
+
+            """def bar():
+                 [i for i in els
+                    async for b in els
+                    for c in b]
+            """,
+
+            """def bar():
+                 [i for i in els
+                    for b in await els]
+            """,
+
+            """def bar():
+                 [i for i in els
+                    for b in els
+                        if await b]
+            """,
+
+            """def bar():
+                 [i for i in await els]
+            """,
+
+            """def bar():
+                 [i for i in els if await i]
+            """,
+
+            """async def foo():
+                await
+            """,
+
+            """async def foo():
+                   def bar(): pass
+                   await = 1
+            """,
+
+            """async def foo():
+
+                   def bar(): pass
+                   await = 1
+            """,
+
+            """async def foo():
+                   def bar(): pass
+                   if 1:
+                       await = 1
+            """,
+
+            """def foo():
+                   async def bar(): pass
+                   if 1:
+                       await a
+            """,
 
-            with self.assertRaisesRegex(Errors.CompileError, 'await.*in comprehen'):
-                exec('async def f():\n\t{0}'.format(comp), ns, ns)
+            """def foo():
+                   async def bar(): pass
+                   await a
+            """,
+
+            """def foo():
+                   def baz(): pass
+                   async def bar(): pass
+                   await a
+            """,
+
+            """def foo():
+                   def baz(): pass
+                   # 456
+                   async def bar(): pass
+                   # 123
+                   await a
+            """,
+
+            """async def foo():
+                   def baz(): pass
+                   # 456
+                   async def bar(): pass
+                   # 123
+                   await = 2
+            """,
+
+            """def foo():
+
+                   def baz(): pass
+
+                   async def bar(): pass
+
+                   await a
+            """,
+
+            """async def foo():
+
+                   def baz(): pass
+
+                   async def bar(): pass
+
+                   await = 2
+            """,
+
+            """async def foo():
+                   def async(): pass
+            """,
+
+            """async def foo():
+                   def await(): pass
+            """,
+
+            """async def foo():
+                   def bar():
+                       await
+            """,
+
+            """async def foo():
+                   return lambda async: await
+            """,
+
+            """async def foo():
+                   return lambda a: await
+            """,
+
+            """await a()""",
+
+            """async def foo(a=await b):
+                   pass
+            """,
+
+            #"""async def foo(a:await b):
+            #       pass
+            #""",
+
+            """def baz():
+                   async def foo(a=await b):
+                       pass
+            """,
+
+            """async def foo(async):
+                   pass
+            """,
+
+            """async def foo():
+                   def bar():
+                        def baz():
+                            async = 1
+            """,
+
+            """async def foo():
+                   def bar():
+                        def baz():
+                            pass
+                        async = 1
+            """,
+
+            """def foo():
+                   async def bar():
+
+                        async def baz():
+                            pass
+
+                        def baz():
+                            42
+
+                        async = 1
+            """,
+
+            """async def foo():
+                   def bar():
+                        def baz():
+                            pass\nawait foo()
+            """,
+
+            """def foo():
+                   def bar():
+                        async def baz():
+                            pass\nawait foo()
+            """,
+
+            """async def foo(await):
+                   pass
+            """,
+
+            """def foo():
+
+                   async def bar(): pass
+
+                   await a
+            """,
+
+            """def foo():
+                   async def bar():
+                        pass\nawait a
+            """]
+
+        for code in samples:
+            with self.subTest(code=code), self.assertRaisesRegex(Errors.CompileError, '.'):
+                compile(code, "", "exec")
+
+    def test_badsyntax_2(self):
+        samples = [
+            """def foo():
+                await = 1
+            """,
+
+            """class Bar:
+                def async(): pass
+            """,
+
+            """class Bar:
+                async = 1
+            """,
+
+            """class async:
+                pass
+            """,
+
+            """class await:
+                pass
+            """,
+
+            """import math as await""",
+
+            """def async():
+                pass""",
+
+            """def foo(*, await=1):
+                pass"""
+
+            """async = 1""",
+
+            # FIXME: cannot currently request Py3 syntax in cython.inline()
+            #"""print(await=1)"""
+        ]
+
+        for code in samples:
+            with self.subTest(code=code):  # , self.assertRaisesRegex(Errors.CompileError, '.'):
+                compile(code, "", "exec")
+
+    def test_badsyntax_3(self):
+        #with self.assertRaises(DeprecationWarning):
+            with warnings.catch_warnings():
+                warnings.simplefilter("error")
+                compile("async = 1", "", "exec")
 
     def test_badsyntax_10(self):
         # Tests for issue 24619
@@ -262,9 +624,9 @@
                    pass
             """,
 
-            """async def foo(a:await b):
-                   pass
-            """,
+            #"""async def foo(a:await b):
+            #       pass
+            #""",
 
             """def baz():
                    async def foo(a=await b):
@@ -437,10 +799,15 @@
 
     @contextlib.contextmanager
     def assertRaisesRegex(self, exc_type, regex):
+        class Holder(object):
+            exception = None
+
+        holder = Holder()
         # the error messages usually don't match, so we just ignore them
         try:
-            yield
-        except exc_type:
+            yield holder
+        except exc_type as exc:
+            holder.exception = exc
             self.assertTrue(True)
         else:
             self.assertTrue(False)
@@ -484,6 +851,10 @@
         def assertIsNotNone(self, value, msg=None):
             self.assertTrue(value is not None, msg)
 
+    if not hasattr(unittest.TestCase, 'assertIsInstance'):
+        def assertIsInstance(self, obj, cls, msg=None):
+            self.assertTrue(isinstance(obj, cls), msg)
+
     def test_gen_1(self):
         def gen(): yield
         self.assertFalse(hasattr(gen, '__await__'))
@@ -513,7 +884,7 @@
         self.assertEqual(run_async__await__(foo()), ([], 10))
 
         def bar(): pass
-        self.assertFalse(bool(bar.__code__.co_flags & 0x80))
+        self.assertFalse(bool(bar.__code__.co_flags & 0x80))  # inspect.CO_COROUTINE
 
     # TODO
     def __test_func_2(self):
@@ -530,7 +901,7 @@
             raise StopIteration
 
         with silence_coro_gc():
-            self.assertRegex(repr(foo()), '^$')
+            self.assertRegex(repr(foo()), '^<[^\s]*coroutine object.* at 0x.*>$')
 
     def test_func_4(self):
         async def foo():
@@ -624,6 +995,7 @@
     def test_func_9(self):
         async def foo(): pass
 
+        gc.collect()
         with self.assertWarnsRegex(
             RuntimeWarning, "coroutine '.*test_func_9.*foo' was never awaited"):
 
@@ -653,7 +1025,7 @@
         self.assertTrue(aw is iter(aw))
         next(aw)
         self.assertEqual(aw.send(10), 100)
-        with self.assertRaises(TypeError):
+        with self.assertRaises(TypeError):   # removed from CPython test suite?
             type(aw).send(None, None)
 
         self.assertEqual(N, 0)
@@ -714,6 +1086,165 @@
                                     "coroutine ignored GeneratorExit"):
             c.close()
 
+    def test_func_15(self):
+        # See https://bugs.python.org/issue25887 for details
+
+        async def spammer():
+            return 'spam'
+        async def reader(coro):
+            return await coro
+
+        spammer_coro = spammer()
+
+        with self.assertRaisesRegex(StopIteration, 'spam'):
+            reader(spammer_coro).send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            reader(spammer_coro).send(None)
+
+    def test_func_16(self):
+        # See https://bugs.python.org/issue25887 for details
+
+        @types_coroutine
+        def nop():
+            yield
+        async def send():
+            await nop()
+            return 'spam'
+        async def read(coro):
+            await nop()
+            return await coro
+
+        spammer = send()
+
+        reader = read(spammer)
+        reader.send(None)
+        reader.send(None)
+        with self.assertRaisesRegex(Exception, 'ham'):
+            reader.throw(Exception('ham'))
+
+        reader = read(spammer)
+        reader.send(None)
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            reader.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            reader.throw(Exception('wat'))
+
+    def test_func_17(self):
+        # See https://bugs.python.org/issue25887 for details
+
+        async def coroutine():
+            return 'spam'
+
+        coro = coroutine()
+        with self.assertRaisesRegex(StopIteration, 'spam'):
+            coro.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            coro.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            coro.throw(Exception('wat'))
+
+        # Closing a coroutine shouldn't raise any exception even if it's
+        # already closed/exhausted (similar to generators)
+        coro.close()
+        coro.close()
+
+    def test_func_18(self):
+        # See https://bugs.python.org/issue25887 for details
+
+        async def coroutine():
+            return 'spam'
+
+        coro = coroutine()
+        await_iter = coro.__await__()
+        it = iter(await_iter)
+
+        with self.assertRaisesRegex(StopIteration, 'spam'):
+            it.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            it.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            # Although the iterator protocol requires iterators to
+            # raise another StopIteration here, we don't want to do
+            # that.  In this particular case, the iterator will raise
+            # a RuntimeError, so that 'yield from' and 'await'
+            # expressions will trigger the error, instead of silently
+            # ignoring the call.
+            next(it)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            it.throw(Exception('wat'))
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    'cannot reuse already awaited coroutine'):
+            it.throw(Exception('wat'))
+
+        # Closing a coroutine shouldn't raise any exception even if it's
+        # already closed/exhausted (similar to generators)
+        it.close()
+        it.close()
+
+    def test_func_19(self):
+        CHK = 0
+
+        @types_coroutine
+        def foo():
+            nonlocal CHK
+            yield
+            try:
+                yield
+            except GeneratorExit:
+                CHK += 1
+
+        async def coroutine():
+            await foo()
+
+        coro = coroutine()
+
+        coro.send(None)
+        coro.send(None)
+
+        self.assertEqual(CHK, 0)
+        coro.close()
+        self.assertEqual(CHK, 1)
+
+        for _ in range(3):
+            # Closing a coroutine shouldn't raise any exception even if it's
+            # already closed/exhausted (similar to generators)
+            coro.close()
+            self.assertEqual(CHK, 1)
+
+    def test_coro_wrapper_send_tuple(self):
+        async def foo():
+            return (10,)
+
+        result = run_async__await__(foo())
+        self.assertEqual(result, ([], (10,)))
+
+    def test_coro_wrapper_send_stop_iterator(self):
+        async def foo():
+            return StopIteration(10)
+
+        result = run_async__await__(foo())
+        self.assertIsInstance(result[1], StopIteration)
+        if sys.version_info >= (3, 3):
+            self.assertEqual(result[1].value, 10)
+        else:
+            self.assertEqual(result[1].args[0], 10)
+
     def test_cr_await(self):
         @types_coroutine
         def a():
@@ -829,7 +1360,7 @@
         class Awaitable(object):
             pass
 
-        async def foo(): return (await Awaitable())
+        async def foo(): return await Awaitable()
 
         with self.assertRaisesRegex(
             TypeError, "object Awaitable can't be used in 'await' expression"):
@@ -899,7 +1430,7 @@
             return await Awaitable()
 
         with self.assertRaisesRegex(
-            TypeError, "__await__\(\) returned a coroutine"):
+            TypeError, r"__await__\(\) returned a coroutine"):
 
             run_async(foo())
 
@@ -949,7 +1480,42 @@
         with self.assertRaises(Marker):
             c.throw(ZeroDivisionError)
 
-    def test_await_iterator(self):
+    def test_await_15(self):
+        @types_coroutine
+        def nop():
+            yield
+
+        async def coroutine():
+            await nop()
+
+        async def waiter(coro):
+            await coro
+
+        coro = coroutine()
+        coro.send(None)
+
+        with self.assertRaisesRegex(RuntimeError,
+                                    "coroutine is being awaited already"):
+            waiter(coro).send(None)
+
+    def test_await_16(self):
+        # See https://bugs.python.org/issue29600 for details.
+
+        async def f():
+            return ValueError()
+
+        async def g():
+            try:
+                raise KeyError
+            except:
+                return await f()
+
+        _, result = run_async(g())
+        if sys.version_info[0] >= 3:
+            self.assertIsNone(result.__context__)
+
+    # removed from CPython ?
+    def __test_await_iterator(self):
         async def foo():
             return 123
 
@@ -1234,7 +1800,8 @@
             run_async(foo())
         self.assertEqual(CNT, 1)
 
-    def test_for_1(self):
+    # old-style pre-Py3.5.2 protocol - no longer supported
+    def __test_for_1(self):
         aiter_calls = 0
 
         class AsyncIter(object):
@@ -1260,7 +1827,7 @@
 
         buffer = []
         async def test1():
-            with ignore_py26(self.assertWarnsRegex(PendingDeprecationWarning, "legacy")):
+            with self.assertWarnsRegex(DeprecationWarning, "legacy"):
                 async for i1, i2 in AsyncIter():
                     buffer.append(i1 + i2)
 
@@ -1274,7 +1841,7 @@
         buffer = []
         async def test2():
             nonlocal buffer
-            with ignore_py26(self.assertWarnsRegex(PendingDeprecationWarning, "legacy")):
+            with self.assertWarnsRegex(DeprecationWarning, "legacy"):
                 async for i in AsyncIter():
                     buffer.append(i[0])
                     if i[0] == 20:
@@ -1293,7 +1860,7 @@
         buffer = []
         async def test3():
             nonlocal buffer
-            with ignore_py26(self.assertWarnsRegex(PendingDeprecationWarning, "legacy")):
+            with self.assertWarnsRegex(DeprecationWarning, "legacy"):
                 async for i in AsyncIter():
                     if i[0] > 20:
                         continue
@@ -1311,7 +1878,7 @@
 
     def test_for_2(self):
         tup = (1, 2, 3)
-        refs_before = sys.getrefcount(tup)
+        refs_before = getrefcount(tup)
 
         async def foo():
             async for i in tup:
@@ -1322,7 +1889,7 @@
 
             run_async(foo())
 
-        self.assertEqual(sys.getrefcount(tup), refs_before)
+        self.assertEqual(getrefcount(tup), refs_before)
 
     def test_for_3(self):
         class I(object):
@@ -1330,7 +1897,7 @@
                 return self
 
         aiter = I()
-        refs_before = sys.getrefcount(aiter)
+        refs_before = getrefcount(aiter)
 
         async def foo():
             async for i in aiter:
@@ -1342,7 +1909,7 @@
 
             run_async(foo())
 
-        self.assertEqual(sys.getrefcount(aiter), refs_before)
+        self.assertEqual(getrefcount(aiter), refs_before)
 
     def test_for_4(self):
         class I(object):
@@ -1353,7 +1920,7 @@
                 return ()
 
         aiter = I()
-        refs_before = sys.getrefcount(aiter)
+        refs_before = getrefcount(aiter)
 
         async def foo():
             async for i in aiter:
@@ -1365,7 +1932,7 @@
 
             run_async(foo())
 
-        self.assertEqual(sys.getrefcount(aiter), refs_before)
+        self.assertEqual(getrefcount(aiter), refs_before)
 
     def test_for_5(self):
         class I(object):
@@ -1376,7 +1943,7 @@
                 return 123
 
         async def foo():
-            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+            with self.assertWarnsRegex(DeprecationWarning, "legacy"):
                 async for i in I():
                     print('never going to happen')
 
@@ -1415,8 +1982,8 @@
 
         manager = Manager()
         iterable = Iterable()
-        mrefs_before = sys.getrefcount(manager)
-        irefs_before = sys.getrefcount(iterable)
+        mrefs_before = getrefcount(manager)
+        irefs_before = getrefcount(iterable)
 
         async def main():
             nonlocal I
@@ -1429,8 +1996,8 @@
         run_async(main())
         self.assertEqual(I, 111011)
 
-        self.assertEqual(sys.getrefcount(manager), mrefs_before)
-        self.assertEqual(sys.getrefcount(iterable), irefs_before)
+        self.assertEqual(getrefcount(manager), mrefs_before)
+        self.assertEqual(getrefcount(iterable), irefs_before)
 
         ##############
 
@@ -1474,14 +2041,15 @@
         run_async(main())
         self.assertEqual(I, 20555255)
 
-    def test_for_7(self):
+    # old-style pre-Py3.5.2 protocol - no longer supported
+    def __test_for_7(self):
         CNT = 0
         class AI(object):
             async def __aiter__(self):
                 1/0
         async def foo():
             nonlocal CNT
-            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+            with self.assertWarnsRegex(DeprecationWarning, "legacy"):
                 async for i in AI():
                     CNT += 1
             CNT += 10
@@ -1491,7 +2059,7 @@
 
     def test_for_8(self):
         CNT = 0
-        class AI:
+        class AI(object):
             def __aiter__(self):
                 1/0
         async def foo():
@@ -1500,7 +2068,7 @@
                 CNT += 1
             CNT += 10
         with self.assertRaises(ZeroDivisionError):
-            run_async(foo())
+            #run_async(foo())
             with warnings.catch_warnings():
                 warnings.simplefilter("error")
                 # Test that if __aiter__ raises an exception it propagates
@@ -1508,39 +2076,314 @@
                 run_async(foo())
         self.assertEqual(CNT, 0)
 
-    @min_py27
-    def test_for_9(self):
-        # Test that PendingDeprecationWarning can safely be converted into
+    # old-style pre-Py3.5.2 protocol - no longer supported
+    def __test_for_9(self):
+        # Test that DeprecationWarning can safely be converted into
         # an exception (__aiter__ should not have a chance to raise
         # a ZeroDivisionError.)
-        class AI:
+        class AI(object):
             async def __aiter__(self):
                 1/0
         async def foo():
             async for i in AI():
                 pass
 
-        with self.assertRaises(PendingDeprecationWarning):
+        with self.assertRaises(DeprecationWarning):
             with warnings.catch_warnings():
                 warnings.simplefilter("error")
                 run_async(foo())
 
-    @min_py27
-    def test_for_10(self):
-        # Test that PendingDeprecationWarning can safely be converted into
+    # old-style pre-Py3.5.2 protocol - no longer supported
+    def __test_for_10(self):
+        # Test that DeprecationWarning can safely be converted into
         # an exception.
-        class AI:
+        class AI(object):
             async def __aiter__(self):
                 pass
         async def foo():
             async for i in AI():
                 pass
 
-        with self.assertRaises(PendingDeprecationWarning):
+        with self.assertRaises(DeprecationWarning):
             with warnings.catch_warnings():
                 warnings.simplefilter("error")
                 run_async(foo())
 
+    def test_for_11(self):
+        class F(object):
+            def __aiter__(self):
+                return self
+            def __anext__(self):
+                return self
+            def __await__(self):
+                1 / 0
+
+        async def main():
+            async for _ in F():
+                pass
+
+        if sys.version_info[0] < 3:
+            with self.assertRaises(ZeroDivisionError) as c:
+                main().send(None)
+        else:
+            with self.assertRaisesRegex(TypeError,
+                                        'an invalid object from __anext__') as c:
+                main().send(None)
+
+            err = c.exception
+            self.assertIsInstance(err.__cause__, ZeroDivisionError)
+
+    # old-style pre-Py3.5.2 protocol - no longer supported
+    def __test_for_12(self):
+        class F(object):
+            def __aiter__(self):
+                return self
+            def __await__(self):
+                1 / 0
+
+        async def main():
+            async for _ in F():
+                pass
+
+        if sys.version_info[0] < 3:
+            with self.assertRaises(ZeroDivisionError) as c:
+                main().send(None)
+        else:
+            with self.assertRaisesRegex(TypeError,
+                                        'an invalid object from __aiter__') as c:
+                main().send(None)
+
+            err = c.exception
+            self.assertIsInstance(err.__cause__, ZeroDivisionError)
+
+    def test_for_tuple(self):
+        class Done(Exception): pass
+
+        class AIter(tuple):
+            i = 0
+            def __aiter__(self):
+                return self
+            async def __anext__(self):
+                if self.i >= len(self):
+                    raise StopAsyncIteration
+                self.i += 1
+                return self[self.i - 1]
+
+        result = []
+        async def foo():
+            async for i in AIter([42]):
+                result.append(i)
+            raise Done
+
+        with self.assertRaises(Done):
+            foo().send(None)
+        self.assertEqual(result, [42])
+
+    def test_for_stop_iteration(self):
+        class Done(Exception): pass
+
+        class AIter(StopIteration):
+            i = 0
+            def __aiter__(self):
+                return self
+            async def __anext__(self):
+                if self.i:
+                    raise StopAsyncIteration
+                self.i += 1
+                if sys.version_info >= (3, 3):
+                    return self.value
+                else:
+                    return self.args[0]
+
+        result = []
+        async def foo():
+            async for i in AIter(42):
+                result.append(i)
+            raise Done
+
+        with self.assertRaises(Done):
+            foo().send(None)
+        self.assertEqual(result, [42])
+
+    def test_comp_1(self):
+        async def f(i):
+            return i
+
+        async def run_list():
+            return [await c for c in [f(1), f(41)]]
+
+        async def run_set():
+            return {await c for c in [f(1), f(41)]}
+
+        async def run_dict1():
+            return {await c: 'a' for c in [f(1), f(41)]}
+
+        async def run_dict2():
+            return {i: await c for i, c in enumerate([f(1), f(41)])}
+
+        self.assertEqual(run_async(run_list()), ([], [1, 41]))
+        self.assertEqual(run_async(run_set()), ([], {1, 41}))
+        self.assertEqual(run_async(run_dict1()), ([], {1: 'a', 41: 'a'}))
+        self.assertEqual(run_async(run_dict2()), ([], {0: 1, 1: 41}))
+
+    def test_comp_2(self):
+        async def f(i):
+            return i
+
+        async def run_list():
+            return [s for c in [f(''), f('abc'), f(''), f(['de', 'fg'])]
+                    for s in await c]
+
+        self.assertEqual(
+            run_async(run_list()),
+            ([], ['a', 'b', 'c', 'de', 'fg']))
+
+        async def run_set():
+            return {d
+                    for c in [f([f([10, 30]),
+                                 f([20])])]
+                    for s in await c
+                    for d in await s}
+
+        self.assertEqual(
+            run_async(run_set()),
+            ([], {10, 20, 30}))
+
+        async def run_set2():
+            return {await s
+                    for c in [f([f(10), f(20)])]
+                    for s in await c}
+
+        self.assertEqual(
+            run_async(run_set2()),
+            ([], {10, 20}))
+
+    def test_comp_3(self):
+        async def f(it):
+            for i in it:
+                yield i
+
+        async def run_list():
+            return [i + 1 async for i in f([10, 20])]
+        self.assertEqual(
+            run_async(run_list()),
+            ([], [11, 21]))
+
+        async def run_set():
+            return {i + 1 async for i in f([10, 20])}
+        self.assertEqual(
+            run_async(run_set()),
+            ([], {11, 21}))
+
+        async def run_dict():
+            return {i + 1: i + 2 async for i in f([10, 20])}
+        self.assertEqual(
+            run_async(run_dict()),
+            ([], {11: 12, 21: 22}))
+
+        async def run_gen():
+            gen = (i + 1 async for i in f([10, 20]))
+            return [g + 100 async for g in gen]
+        self.assertEqual(
+            run_async(run_gen()),
+            ([], [111, 121]))
+
+    def test_comp_4(self):
+        async def f(it):
+            for i in it:
+                yield i
+
+        async def run_list():
+            return [i + 1 async for i in f([10, 20]) if i > 10]
+        self.assertEqual(
+            run_async(run_list()),
+            ([], [21]))
+
+        async def run_set():
+            return {i + 1 async for i in f([10, 20]) if i > 10}
+        self.assertEqual(
+            run_async(run_set()),
+            ([], {21}))
+
+        async def run_dict():
+            return {i + 1: i + 2 async for i in f([10, 20]) if i > 10}
+        self.assertEqual(
+            run_async(run_dict()),
+            ([], {21: 22}))
+
+        async def run_gen():
+            gen = (i + 1 async for i in f([10, 20]) if i > 10)
+            return [g + 100 async for g in gen]
+        self.assertEqual(
+            run_async(run_gen()),
+            ([], [121]))
+
+    def test_comp_5(self):
+        async def f(it):
+            for i in it:
+                yield i
+
+        async def run_list():
+            return [i + 1 for pair in ([10, 20], [30, 40]) if pair[0] > 10
+                    async for i in f(pair) if i > 30]
+        self.assertEqual(
+            run_async(run_list()),
+            ([], [41]))
+
+    def test_comp_6(self):
+        async def f(it):
+            for i in it:
+                yield i
+
+        async def run_list():
+            return [i + 1 async for seq in f([(10, 20), (30,)])
+                    for i in seq]
+
+        self.assertEqual(
+            run_async(run_list()),
+            ([], [11, 21, 31]))
+
+    def test_comp_7(self):
+        async def f():
+            yield 1
+            yield 2
+            raise Exception('aaa')
+
+        async def run_list():
+            return [i async for i in f()]
+
+        with self.assertRaisesRegex(Exception, 'aaa'):
+            run_async(run_list())
+
+    def test_comp_8(self):
+        async def f():
+            return [i for i in [1, 2, 3]]
+
+        self.assertEqual(
+            run_async(f()),
+            ([], [1, 2, 3]))
+
+    def test_comp_9(self):
+        async def gen():
+            yield 1
+            yield 2
+        async def f():
+            l = [i async for i in gen()]
+            return [i for i in l]
+
+        self.assertEqual(
+            run_async(f()),
+            ([], [1, 2]))
+
+    def test_comp_10(self):
+        async def f():
+            xx = {i for i in [1, 2, 3]}
+            return {x: x for x in xx}
+
+        self.assertEqual(
+            run_async(f()),
+            ([], {1: 1, 2: 2, 3: 3}))
+
     def test_copy(self):
         async def func(): pass
         coro = func()
@@ -1569,6 +2412,18 @@
         finally:
             aw.close()
 
+    @no_pypy
+    def test_fatal_coro_warning(self):
+        # Issue 27811
+        async def func(): pass
+
+        gc.collect()
+        with warnings.catch_warnings(), captured_stderr() as stderr:
+            warnings.filterwarnings("error")
+            func()
+            gc.collect()
+        self.assertIn("was never awaited", stderr.getvalue())
+
 
 class CoroAsyncIOCompatTest(unittest.TestCase):
 
@@ -1609,6 +2464,31 @@
 
         self.assertEqual(buffer, [1, 2, 'MyException'])
 
+    def test_asyncio_cython_crash_gh1999(self):
+        async def await_future(loop):
+            fut = loop.create_future()
+            loop.call_later(1, lambda: fut.set_result(1))
+            await fut
+
+        async def delegate_to_await_future(loop):
+            await await_future(loop)
+
+        ns = {}
+        __builtins__.exec("""
+        async def call(loop, await_func):  # requires Py3.5+
+            await await_func(loop)
+        """.strip(), ns, ns)
+        call = ns['call']
+
+        import asyncio
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        try:
+            loop.run_until_complete(call(loop, delegate_to_await_future))
+        finally:
+            loop.close()
+            asyncio.set_event_loop(None)
+
 
 class SysSetCoroWrapperTest(unittest.TestCase):
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_exceptions.pyx cython-0.20.1+1~202203241016-9537/tests/run/test_exceptions.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/test_exceptions.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_exceptions.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,1380 @@
+# Python test set -- part 5, built-in exceptions
+# Copied from CPython 3.7.
+
+# cython: language_level=3
+# mode: run
+# tag: generator, exception, tryfinally, tryexcept, test.support
+
+import copy
+import os
+import sys
+import unittest
+import pickle
+import weakref
+import errno
+
+from test.support import (captured_stderr, check_impl_detail, gc_collect,
+                          # no_tracing, cpython_only,
+                          script_helper, SuppressCrashReport)
+try:
+    from test.support.os_helper import TESTFN, unlink
+    from test.support.warnings_helper import check_warnings
+    from test.support.import_helper import import_module
+except ImportError:
+    # Python 3.9 and older
+    from test.support import check_warnings, TESTFN, unlink, import_module
+
+no_tracing = unittest.skip("For nested functions, Cython generates a C call without recursion checks.")
+
+cpython_only = unittest.skip("Tests for _testcapi or Python error messages make no sense here.")
+
+
+class NaiveException(Exception):
+    def __init__(self, x):
+        self.x = x
+
+class SlottedNaiveException(Exception):
+    __slots__ = ('x',)
+    def __init__(self, x):
+        self.x = x
+
+class BrokenStrException(Exception):
+    def __str__(self):
+        raise Exception("str() is broken")
+
+# XXX This is not really enough, each *operation* should be tested!
+
+class ExceptionTests(unittest.TestCase):
+
+    def raise_catch(self, exc, excname):
+        try:
+            raise exc("spam")
+        except exc as err:
+            buf1 = str(err)
+        try:
+            raise exc("spam")
+        except exc as err:
+            buf2 = str(err)
+        self.assertEqual(buf1, buf2)
+        self.assertEqual(exc.__name__, excname)
+
+    def testRaising(self):
+        self.raise_catch(AttributeError, "AttributeError")
+        self.assertRaises(AttributeError, getattr, sys, "undefined_attribute")
+
+        self.raise_catch(EOFError, "EOFError")
+        fp = open(TESTFN, 'w')
+        fp.close()
+        fp = open(TESTFN, 'r')
+        savestdin = sys.stdin
+        try:
+            try:
+                import marshal
+                marshal.loads(b'')
+            except EOFError:
+                pass
+        finally:
+            sys.stdin = savestdin
+            fp.close()
+            unlink(TESTFN)
+
+        self.raise_catch(OSError, "OSError")
+        self.assertRaises(OSError, open, 'this file does not exist', 'r')
+
+        self.raise_catch(ImportError, "ImportError")
+        self.assertRaises(ImportError, __import__, "undefined_module")
+
+        self.raise_catch(IndexError, "IndexError")
+        x = []
+        self.assertRaises(IndexError, x.__getitem__, 10)
+
+        self.raise_catch(KeyError, "KeyError")
+        x = {}
+        self.assertRaises(KeyError, x.__getitem__, 'key')
+
+        self.raise_catch(KeyboardInterrupt, "KeyboardInterrupt")
+
+        self.raise_catch(MemoryError, "MemoryError")
+
+        self.raise_catch(NameError, "NameError")
+        #try: x = undefined_variable
+        #except NameError: pass
+
+        self.raise_catch(OverflowError, "OverflowError")
+        x = 1
+        for dummy in range(128):
+            x += x  # this simply shouldn't blow up
+
+        self.raise_catch(RuntimeError, "RuntimeError")
+        self.raise_catch(RecursionError, "RecursionError")
+
+        self.raise_catch(SyntaxError, "SyntaxError")
+        try: exec('/\n')
+        except SyntaxError: pass
+
+        self.raise_catch(IndentationError, "IndentationError")
+
+        self.raise_catch(TabError, "TabError")
+        try: compile("try:\n\t1/0\n    \t1/0\nfinally:\n pass\n",
+                     '', 'exec')
+        except TabError: pass
+        else: self.fail("TabError not raised")
+
+        self.raise_catch(SystemError, "SystemError")
+
+        self.raise_catch(SystemExit, "SystemExit")
+        self.assertRaises(SystemExit, sys.exit, 0)
+
+        self.raise_catch(TypeError, "TypeError")
+        try: [] + ()
+        except TypeError: pass
+
+        self.raise_catch(ValueError, "ValueError")
+        self.assertRaises(ValueError, chr, 17<<16)
+
+        ZERO = 0
+        self.raise_catch(ZeroDivisionError, "ZeroDivisionError")
+        try: x = 1/ZERO
+        except ZeroDivisionError: pass
+
+        self.raise_catch(Exception, "Exception")
+        try: x = 1/ZERO
+        except Exception as e: pass
+
+        self.raise_catch(StopAsyncIteration, "StopAsyncIteration")
+
+    @cpython_only
+    def testSyntaxErrorMessage(self):
+        # make sure the right exception message is raised for each of
+        # these code fragments
+
+        def ckmsg(src, msg):
+            try:
+                compile(src, '', 'exec')
+            except SyntaxError as e:
+                if e.msg != msg:
+                    self.fail("expected %s, got %s" % (msg, e.msg))
+            else:
+                self.fail("failed to get expected SyntaxError")
+
+        s = '''if 1:
+        try:
+            continue
+        except:
+            pass'''
+
+        ckmsg(s, "'continue' not properly in loop")
+        ckmsg("continue\n", "'continue' not properly in loop")
+
+    @cpython_only
+    def testSyntaxErrorMissingParens(self):
+        def ckmsg(src, msg, exception=SyntaxError):
+            try:
+                compile(src, '', 'exec')
+            except exception as e:
+                if e.msg != msg and sys.version_info >= (3, 6):
+                    self.fail("expected %s, got %s" % (msg, e.msg))
+            else:
+                self.fail("failed to get expected SyntaxError")
+
+        s = '''print "old style"'''
+        ckmsg(s, "Missing parentheses in call to 'print'. "
+                 "Did you mean print(\"old style\")?")
+
+        s = '''print "old style",'''
+        ckmsg(s, "Missing parentheses in call to 'print'. "
+                 "Did you mean print(\"old style\", end=\" \")?")
+
+        s = '''exec "old style"'''
+        ckmsg(s, "Missing parentheses in call to 'exec'")
+
+        # should not apply to subclasses, see issue #31161
+        s = '''if True:\nprint "No indent"'''
+        ckmsg(s, "expected an indented block", IndentationError)
+
+        s = '''if True:\n        print()\n\texec "mixed tabs and spaces"'''
+        ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError)
+
+    @cpython_only
+    def testSyntaxErrorOffset(self):
+        def check(src, lineno, offset):
+            with self.assertRaises(SyntaxError) as cm:
+                compile(src, '', 'exec')
+            self.assertEqual(cm.exception.lineno, lineno)
+            self.assertEqual(cm.exception.offset, offset)
+
+        check('def fact(x):\n\treturn x!\n', 2, 10)
+        check('1 +\n', 1, 4)
+        check('def spam():\n  print(1)\n print(2)', 3, 10)
+        check('Python = "Python" +', 1, 20)
+        check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20)
+
+    @cpython_only
+    def testSettingException(self):
+        # test that setting an exception at the C level works even if the
+        # exception object can't be constructed.
+
+        class BadException(Exception):
+            def __init__(self_):
+                raise RuntimeError("can't instantiate BadException")
+
+        class InvalidException:
+            pass
+
+        def test_capi1():
+            import _testcapi
+            try:
+                _testcapi.raise_exception(BadException, 1)
+            except TypeError as err:
+                exc, err, tb = sys.exc_info()
+                co = tb.tb_frame.f_code
+                self.assertEqual(co.co_name, "test_capi1")
+                self.assertTrue(co.co_filename.endswith('test_exceptions.py'))
+            else:
+                self.fail("Expected exception")
+
+        def test_capi2():
+            import _testcapi
+            try:
+                _testcapi.raise_exception(BadException, 0)
+            except RuntimeError as err:
+                exc, err, tb = sys.exc_info()
+                co = tb.tb_frame.f_code
+                self.assertEqual(co.co_name, "__init__")
+                self.assertTrue(co.co_filename.endswith('test_exceptions.py'))
+                co2 = tb.tb_frame.f_back.f_code
+                self.assertEqual(co2.co_name, "test_capi2")
+            else:
+                self.fail("Expected exception")
+
+        def test_capi3():
+            import _testcapi
+            self.assertRaises(SystemError, _testcapi.raise_exception,
+                              InvalidException, 1)
+
+        if not sys.platform.startswith('java'):
+            test_capi1()
+            test_capi2()
+            test_capi3()
+
+    def test_WindowsError(self):
+        try:
+            WindowsError
+        except NameError:
+            pass
+        else:
+            self.assertIs(WindowsError, OSError)
+            self.assertEqual(str(OSError(1001)), "1001")
+            self.assertEqual(str(OSError(1001, "message")),
+                             "[Errno 1001] message")
+            # POSIX errno (9 aka EBADF) is untranslated
+            w = OSError(9, 'foo', 'bar')
+            self.assertEqual(w.errno, 9)
+            self.assertEqual(w.winerror, None)
+            self.assertEqual(str(w), "[Errno 9] foo: 'bar'")
+            # ERROR_PATH_NOT_FOUND (win error 3) becomes ENOENT (2)
+            w = OSError(0, 'foo', 'bar', 3)
+            self.assertEqual(w.errno, 2)
+            self.assertEqual(w.winerror, 3)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, 'bar')
+            self.assertEqual(w.filename2, None)
+            self.assertEqual(str(w), "[WinError 3] foo: 'bar'")
+            # Unknown win error becomes EINVAL (22)
+            w = OSError(0, 'foo', None, 1001)
+            self.assertEqual(w.errno, 22)
+            self.assertEqual(w.winerror, 1001)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, None)
+            self.assertEqual(w.filename2, None)
+            self.assertEqual(str(w), "[WinError 1001] foo")
+            # Non-numeric "errno"
+            w = OSError('bar', 'foo')
+            self.assertEqual(w.errno, 'bar')
+            self.assertEqual(w.winerror, None)
+            self.assertEqual(w.strerror, 'foo')
+            self.assertEqual(w.filename, None)
+            self.assertEqual(w.filename2, None)
+
+    @unittest.skipUnless(sys.platform == 'win32',
+                         'test specific to Windows')
+    def test_windows_message(self):
+        """Should fill in unknown error code in Windows error message"""
+        ctypes = import_module('ctypes')
+        # this error code has no message, Python formats it as hexadecimal
+        code = 3765269347
+        with self.assertRaisesRegex(OSError, 'Windows Error 0x%x' % code):
+            ctypes.pythonapi.PyErr_SetFromWindowsErr(code)
+
+    def testAttributes(self):
+        # test that exception attributes are happy
+
+        exceptionList = [
+            (BaseException, (), {'args' : ()}),
+            (BaseException, (1, ), {'args' : (1,)}),
+            (BaseException, ('foo',),
+                {'args' : ('foo',)}),
+            (BaseException, ('foo', 1),
+                {'args' : ('foo', 1)}),
+            (SystemExit, ('foo',),
+                {'args' : ('foo',), 'code' : 'foo'}),
+            (OSError, ('foo',),
+                {'args' : ('foo',), 'filename' : None, 'filename2' : None,
+                 'errno' : None, 'strerror' : None}),
+            (OSError, ('foo', 'bar'),
+                {'args' : ('foo', 'bar'),
+                 'filename' : None, 'filename2' : None,
+                 'errno' : 'foo', 'strerror' : 'bar'}),
+            (OSError, ('foo', 'bar', 'baz'),
+                {'args' : ('foo', 'bar'),
+                 'filename' : 'baz', 'filename2' : None,
+                 'errno' : 'foo', 'strerror' : 'bar'}),
+            (OSError, ('foo', 'bar', 'baz', None, 'quux'),
+                {'args' : ('foo', 'bar'), 'filename' : 'baz', 'filename2': 'quux'}),
+            (OSError, ('errnoStr', 'strErrorStr', 'filenameStr'),
+                {'args' : ('errnoStr', 'strErrorStr'),
+                 'strerror' : 'strErrorStr', 'errno' : 'errnoStr',
+                 'filename' : 'filenameStr'}),
+            (OSError, (1, 'strErrorStr', 'filenameStr'),
+                {'args' : (1, 'strErrorStr'), 'errno' : 1,
+                 'strerror' : 'strErrorStr',
+                 'filename' : 'filenameStr', 'filename2' : None}),
+            (SyntaxError, (), {'msg' : None, 'text' : None,
+                'filename' : None, 'lineno' : None, 'offset' : None,
+                'print_file_and_line' : None}),
+            (SyntaxError, ('msgStr',),
+                {'args' : ('msgStr',), 'text' : None,
+                 'print_file_and_line' : None, 'msg' : 'msgStr',
+                 'filename' : None, 'lineno' : None, 'offset' : None}),
+            (SyntaxError, ('msgStr', ('filenameStr', 'linenoStr', 'offsetStr',
+                           'textStr')),
+                {'offset' : 'offsetStr', 'text' : 'textStr',
+                 'args' : ('msgStr', ('filenameStr', 'linenoStr',
+                                      'offsetStr', 'textStr')),
+                 'print_file_and_line' : None, 'msg' : 'msgStr',
+                 'filename' : 'filenameStr', 'lineno' : 'linenoStr'}),
+            (SyntaxError, ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr',
+                           'textStr', 'print_file_and_lineStr'),
+                {'text' : None,
+                 'args' : ('msgStr', 'filenameStr', 'linenoStr', 'offsetStr',
+                           'textStr', 'print_file_and_lineStr'),
+                 'print_file_and_line' : None, 'msg' : 'msgStr',
+                 'filename' : None, 'lineno' : None, 'offset' : None}),
+            (UnicodeError, (), {'args' : (),}),
+            (UnicodeEncodeError, ('ascii', 'a', 0, 1,
+                                  'ordinal not in range'),
+                {'args' : ('ascii', 'a', 0, 1,
+                                           'ordinal not in range'),
+                 'encoding' : 'ascii', 'object' : 'a',
+                 'start' : 0, 'reason' : 'ordinal not in range'}),
+            (UnicodeDecodeError, ('ascii', bytearray(b'\xff'), 0, 1,
+                                  'ordinal not in range'),
+                {'args' : ('ascii', bytearray(b'\xff'), 0, 1,
+                                           'ordinal not in range'),
+                 'encoding' : 'ascii', 'object' : b'\xff',
+                 'start' : 0, 'reason' : 'ordinal not in range'}),
+            (UnicodeDecodeError, ('ascii', b'\xff', 0, 1,
+                                  'ordinal not in range'),
+                {'args' : ('ascii', b'\xff', 0, 1,
+                                           'ordinal not in range'),
+                 'encoding' : 'ascii', 'object' : b'\xff',
+                 'start' : 0, 'reason' : 'ordinal not in range'}),
+            (UnicodeTranslateError, ("\u3042", 0, 1, "ouch"),
+                {'args' : ('\u3042', 0, 1, 'ouch'),
+                 'object' : '\u3042', 'reason' : 'ouch',
+                 'start' : 0, 'end' : 1}),
+            (NaiveException, ('foo',),
+                {'args': ('foo',), 'x': 'foo'}),
+            (SlottedNaiveException, ('foo',),
+                {'args': ('foo',), 'x': 'foo'}),
+        ]
+        try:
+            # More tests are in test_WindowsError
+            exceptionList.append(
+                (WindowsError, (1, 'strErrorStr', 'filenameStr'),
+                    {'args' : (1, 'strErrorStr'),
+                     'strerror' : 'strErrorStr', 'winerror' : None,
+                     'errno' : 1,
+                     'filename' : 'filenameStr', 'filename2' : None})
+            )
+        except NameError:
+            pass
+
+        for exc, args, expected in exceptionList:
+            try:
+                e = exc(*args)
+            except:
+                print("\nexc=%r, args=%r" % (exc, args), file=sys.stderr)
+                raise
+            else:
+                # Verify module name
+                if not type(e).__name__.endswith('NaiveException'):
+                    self.assertEqual(type(e).__module__, 'builtins')
+                # Verify no ref leaks in Exc_str()
+                s = str(e)
+                for checkArgName in expected:
+                    value = getattr(e, checkArgName)
+                    self.assertEqual(repr(value),
+                                     repr(expected[checkArgName]),
+                                     '%r.%s == %r, expected %r' % (
+                                     e, checkArgName,
+                                     value, expected[checkArgName]))
+
+                # test for pickling support
+                for p in [pickle]:
+                    for protocol in range(p.HIGHEST_PROTOCOL + 1):
+                        s = p.dumps(e, protocol)
+                        new = p.loads(s)
+                        for checkArgName in expected:
+                            got = repr(getattr(new, checkArgName))
+                            want = repr(expected[checkArgName])
+                            self.assertEqual(got, want,
+                                             'pickled "%r", attribute "%s' %
+                                             (e, checkArgName))
+
+    def testWithTraceback(self):
+        try:
+            raise IndexError(4)
+        except:
+            tb = sys.exc_info()[2]
+
+        e = BaseException().with_traceback(tb)
+        self.assertIsInstance(e, BaseException)
+        self.assertEqual(e.__traceback__, tb)
+
+        e = IndexError(5).with_traceback(tb)
+        self.assertIsInstance(e, IndexError)
+        self.assertEqual(e.__traceback__, tb)
+
+        class MyException(Exception):
+            pass
+
+        e = MyException().with_traceback(tb)
+        self.assertIsInstance(e, MyException)
+        self.assertEqual(e.__traceback__, tb)
+
+    def testInvalidTraceback(self):
+        try:
+            Exception().__traceback__ = 5
+        except TypeError as e:
+            self.assertIn("__traceback__ must be a traceback", str(e))
+        else:
+            self.fail("No exception raised")
+
+    def testInvalidAttrs(self):
+        self.assertRaises(TypeError, setattr, Exception(), '__cause__', 1)
+        self.assertRaises(TypeError, delattr, Exception(), '__cause__')
+        self.assertRaises(TypeError, setattr, Exception(), '__context__', 1)
+        self.assertRaises(TypeError, delattr, Exception(), '__context__')
+
+    def testNoneClearsTracebackAttr(self):
+        try:
+            raise IndexError(4)
+        except:
+            tb = sys.exc_info()[2]
+
+        e = Exception()
+        e.__traceback__ = tb
+        e.__traceback__ = None
+        self.assertEqual(e.__traceback__, None)
+
+    def testChainingAttrs(self):
+        e = Exception()
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+
+        e = TypeError()
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+
+        class MyException(OSError):
+            pass
+
+        e = MyException()
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+
+    def testChainingDescriptors(self):
+        try:
+            raise Exception()
+        except Exception as exc:
+            e = exc
+
+        self.assertIsNone(e.__context__)
+        self.assertIsNone(e.__cause__)
+        self.assertFalse(e.__suppress_context__)
+
+        e.__context__ = NameError()
+        e.__cause__ = None
+        self.assertIsInstance(e.__context__, NameError)
+        self.assertIsNone(e.__cause__)
+        self.assertTrue(e.__suppress_context__)
+        e.__suppress_context__ = False
+        self.assertFalse(e.__suppress_context__)
+
+    def testKeywordArgs(self):
+        # test that builtin exception don't take keyword args,
+        # but user-defined subclasses can if they want
+        self.assertRaises(TypeError, BaseException, a=1)
+
+        class DerivedException(BaseException):
+            def __init__(self, fancy_arg):
+                BaseException.__init__(self)
+                self.fancy_arg = fancy_arg
+
+        x = DerivedException(fancy_arg=42)
+        self.assertEqual(x.fancy_arg, 42)
+
+    @no_tracing
+    def testInfiniteRecursion(self):
+        def f():
+            return f()
+        self.assertRaises(RecursionError, f)
+
+        def g():
+            try:
+                return g()
+            except ValueError:
+                return -1
+        self.assertRaises(RecursionError, g)
+
+    def test_str(self):
+        # Make sure both instances and classes have a str representation.
+        self.assertTrue(str(Exception))
+        self.assertTrue(str(Exception('a')))
+        self.assertTrue(str(Exception('a', 'b')))
+
+    def testExceptionCleanupNames(self):
+        # Make sure the local variable bound to the exception instance by
+        # an "except" statement is only visible inside the except block.
+        try:
+            raise Exception()
+        except Exception as e:
+            self.assertTrue(e)
+            del e
+        self.assertNotIn('e', locals())
+
+    def testExceptionCleanupState(self):
+        # Make sure exception state is cleaned up as soon as the except
+        # block is left. See #2507
+
+        class MyException(Exception):
+            def __init__(self, obj):
+                self.obj = obj
+        class MyObj:
+            pass
+
+        def inner_raising_func():
+            # Create some references in exception value and traceback
+            local_ref = obj
+            raise MyException(obj)
+
+        # Qualified "except" with "as"
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except MyException as e:
+            pass
+        obj = None
+        obj = wr()
+        self.assertIsNone(obj)
+
+        # Qualified "except" without "as"
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except MyException:
+            pass
+        obj = None
+        obj = wr()
+        self.assertIsNone(obj)
+
+        # Bare "except"
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except:
+            pass
+        obj = None
+        obj = wr()
+        self.assertIsNone(obj)
+
+        # "except" with premature block leave
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        for i in [0]:
+            try:
+                inner_raising_func()
+            except:
+                break
+        obj = None
+        obj = wr()
+        self.assertIsNone(obj)
+
+        # "except" block raising another exception
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            try:
+                inner_raising_func()
+            except:
+                raise KeyError
+        except KeyError as e:
+            # We want to test that the except block above got rid of
+            # the exception raised in inner_raising_func(), but it
+            # also ends up in the __context__ of the KeyError, so we
+            # must clear the latter manually for our test to succeed.
+            e.__context__ = None
+            obj = None
+            obj = wr()
+            # guarantee no ref cycles on CPython (don't gc_collect)
+            if check_impl_detail(cpython=False):
+                gc_collect()
+            self.assertIsNone(obj)
+
+        # Some complicated construct
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        try:
+            inner_raising_func()
+        except MyException:
+            try:
+                try:
+                    raise
+                finally:
+                    raise
+            except MyException:
+                pass
+        obj = None
+        if check_impl_detail(cpython=False):
+            gc_collect()
+        obj = wr()
+        self.assertIsNone(obj)
+
+        # Inside an exception-silencing "with" block
+        class Context:
+            def __enter__(self):
+                return self
+            def __exit__ (self, exc_type, exc_value, exc_tb):
+                return True
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        with Context():
+            inner_raising_func()
+        obj = None
+        if check_impl_detail(cpython=False):
+            gc_collect()
+        obj = wr()
+        self.assertIsNone(obj)
+
+    '''
+    # This is currently a compile error in Cython-3, although it works in Cython-2 => could also work in Cy-3.
+    def test_exception_target_in_nested_scope(self):
+        # issue 4617: This used to raise a SyntaxError
+        # "can not delete variable 'e' referenced in nested scope"
+        def print_error():
+            e
+        ZERO = 0
+        try:
+            1/ZERO
+        except Exception as e:
+            print_error()
+            # implicit "del e" here
+    '''
+
+    def test_generator_leaking(self):
+        # Test that generator exception state doesn't leak into the calling
+        # frame
+        def yield_raise():
+            try:
+                raise KeyError("caught")
+            except KeyError:
+                yield sys.exc_info()[0]
+                yield sys.exc_info()[0]
+            yield sys.exc_info()[0]
+        g = yield_raise()
+        self.assertEqual(next(g), KeyError)
+        self.assertEqual(sys.exc_info()[0], None)
+        self.assertEqual(next(g), KeyError)
+        self.assertEqual(sys.exc_info()[0], None)
+        self.assertEqual(next(g), None)
+
+        # Same test, but inside an exception handler
+        try:
+            raise TypeError("foo")
+        except TypeError:
+            g = yield_raise()
+            self.assertEqual(next(g), KeyError)
+            self.assertEqual(sys.exc_info()[0], TypeError)
+            self.assertEqual(next(g), KeyError)
+            self.assertEqual(sys.exc_info()[0], TypeError)
+            self.assertEqual(next(g), TypeError)
+            del g
+            self.assertEqual(sys.exc_info()[0], TypeError)
+
+    def test_generator_leaking2(self):
+        # See issue 12475.
+        def g():
+            yield
+        try:
+            raise RuntimeError
+        except RuntimeError:
+            it = g()
+            next(it)
+        try:
+            next(it)
+        except StopIteration:
+            pass
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_generator_leaking3(self):
+        # See issue #23353.  When gen.throw() is called, the caller's
+        # exception state should be save and restored.
+        def g():
+            try:
+                yield
+            except ZeroDivisionError:
+                yield sys.exc_info()[1]
+        it = g()
+        next(it)
+        ZERO = 0
+        try:
+            1/ZERO
+        except ZeroDivisionError as e:
+            self.assertIs(sys.exc_info()[1], e)
+            gen_exc = it.throw(e)
+            self.assertIs(sys.exc_info()[1], e)
+            self.assertIs(gen_exc, e)
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_generator_leaking4(self):
+        # See issue #23353.  When an exception is raised by a generator,
+        # the caller's exception state should still be restored.
+        def g():
+            ZERO = 0
+            try:
+                1/ZERO
+            except ZeroDivisionError:
+                yield sys.exc_info()[0]
+                raise
+        it = g()
+        try:
+            raise TypeError
+        except TypeError:
+            # The caller's exception state (TypeError) is temporarily
+            # saved in the generator.
+            tp = next(it)
+        self.assertIs(tp, ZeroDivisionError)
+        try:
+            next(it)
+            # We can't check it immediately, but while next() returns
+            # with an exception, it shouldn't have restored the old
+            # exception state (TypeError).
+        except ZeroDivisionError as e:
+            self.assertIs(sys.exc_info()[1], e)
+        # We used to find TypeError here.
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def test_generator_doesnt_retain_old_exc(self):
+        def g():
+            self.assertIsInstance(sys.exc_info()[1], RuntimeError)
+            yield
+            self.assertEqual(sys.exc_info(), (None, None, None))
+        it = g()
+        try:
+            raise RuntimeError
+        except RuntimeError:
+            next(it)
+        self.assertRaises(StopIteration, next, it)
+
+    def test_generator_finalizing_and_exc_info(self):
+        # See #7173
+        def simple_gen():
+            yield 1
+        def run_gen():
+            gen = simple_gen()
+            try:
+                raise RuntimeError
+            except RuntimeError:
+                return next(gen)
+        run_gen()
+        gc_collect()
+        self.assertEqual(sys.exc_info(), (None, None, None))
+
+    def _check_generator_cleanup_exc_state(self, testfunc):
+        # Issue #12791: exception state is cleaned up as soon as a generator
+        # is closed (reference cycles are broken).
+        class MyException(Exception):
+            def __init__(self, obj):
+                self.obj = obj
+        class MyObj:
+            pass
+
+        def raising_gen():
+            try:
+                raise MyException(obj)
+            except MyException:
+                yield
+
+        obj = MyObj()
+        wr = weakref.ref(obj)
+        g = raising_gen()
+        next(g)
+        testfunc(g)
+        g = obj = None
+        obj = wr()
+        self.assertIsNone(obj)
+
+    def test_generator_throw_cleanup_exc_state(self):
+        def do_throw(g):
+            try:
+                g.throw(RuntimeError())
+            except RuntimeError:
+                pass
+        self._check_generator_cleanup_exc_state(do_throw)
+
+    def test_generator_close_cleanup_exc_state(self):
+        def do_close(g):
+            g.close()
+        self._check_generator_cleanup_exc_state(do_close)
+
+    def test_generator_del_cleanup_exc_state(self):
+        def do_del(g):
+            g = None
+        self._check_generator_cleanup_exc_state(do_del)
+
+    def test_generator_next_cleanup_exc_state(self):
+        def do_next(g):
+            try:
+                next(g)
+            except StopIteration:
+                pass
+            else:
+                self.fail("should have raised StopIteration")
+        self._check_generator_cleanup_exc_state(do_next)
+
+    def test_generator_send_cleanup_exc_state(self):
+        def do_send(g):
+            try:
+                g.send(None)
+            except StopIteration:
+                pass
+            else:
+                self.fail("should have raised StopIteration")
+        self._check_generator_cleanup_exc_state(do_send)
+
+    def test_3114(self):
+        # Bug #3114: in its destructor, MyObject retrieves a pointer to
+        # obsolete and/or deallocated objects.
+        class MyObject:
+            def __del__(self):
+                nonlocal e
+                e = sys.exc_info()
+        e = ()
+        try:
+            raise Exception(MyObject())
+        except:
+            pass
+        self.assertEqual(e, (None, None, None))
+
+    def test_unicode_change_attributes(self):
+        # See issue 7309. This was a crasher.
+
+        u = UnicodeEncodeError('baz', 'xxxxx', 1, 5, 'foo')
+        self.assertEqual(str(u), "'baz' codec can't encode characters in position 1-4: foo")
+        u.end = 2
+        self.assertEqual(str(u), "'baz' codec can't encode character '\\x78' in position 1: foo")
+        u.end = 5
+        u.reason = 0x345345345345345345
+        self.assertEqual(str(u), "'baz' codec can't encode characters in position 1-4: 965230951443685724997")
+        u.encoding = 4000
+        self.assertEqual(str(u), "'4000' codec can't encode characters in position 1-4: 965230951443685724997")
+        u.start = 1000
+        self.assertEqual(str(u), "'4000' codec can't encode characters in position 1000-4: 965230951443685724997")
+
+        u = UnicodeDecodeError('baz', b'xxxxx', 1, 5, 'foo')
+        self.assertEqual(str(u), "'baz' codec can't decode bytes in position 1-4: foo")
+        u.end = 2
+        self.assertEqual(str(u), "'baz' codec can't decode byte 0x78 in position 1: foo")
+        u.end = 5
+        u.reason = 0x345345345345345345
+        self.assertEqual(str(u), "'baz' codec can't decode bytes in position 1-4: 965230951443685724997")
+        u.encoding = 4000
+        self.assertEqual(str(u), "'4000' codec can't decode bytes in position 1-4: 965230951443685724997")
+        u.start = 1000
+        self.assertEqual(str(u), "'4000' codec can't decode bytes in position 1000-4: 965230951443685724997")
+
+        u = UnicodeTranslateError('xxxx', 1, 5, 'foo')
+        self.assertEqual(str(u), "can't translate characters in position 1-4: foo")
+        u.end = 2
+        self.assertEqual(str(u), "can't translate character '\\x78' in position 1: foo")
+        u.end = 5
+        u.reason = 0x345345345345345345
+        self.assertEqual(str(u), "can't translate characters in position 1-4: 965230951443685724997")
+        u.start = 1000
+        self.assertEqual(str(u), "can't translate characters in position 1000-4: 965230951443685724997")
+
+    def test_unicode_errors_no_object(self):
+        # See issue #21134.
+        klasses = UnicodeEncodeError, UnicodeDecodeError, UnicodeTranslateError
+        for klass in klasses:
+            self.assertEqual(str(klass.__new__(klass)), "")
+
+    @no_tracing
+    def test_badisinstance(self):
+        # Bug #2542: if issubclass(e, MyException) raises an exception,
+        # it should be ignored
+        class Meta(type):
+            def __subclasscheck__(cls, subclass):
+                raise ValueError()
+        class MyException(Exception, metaclass=Meta):
+            pass
+
+        with captured_stderr() as stderr:
+            try:
+                raise KeyError()
+            except MyException as e:
+                self.fail("exception should not be a MyException")
+            except KeyError:
+                pass
+            except:
+                self.fail("Should have raised KeyError")
+            else:
+                self.fail("Should have raised KeyError")
+
+        def g():
+            try:
+                return g()
+            except RecursionError:
+                return sys.exc_info()
+        e, v, tb = g()
+        self.assertIsInstance(v, RecursionError, type(v))
+        self.assertIn("maximum recursion depth exceeded", str(v))
+
+    @cpython_only
+    def test_recursion_normalizing_exception(self):
+        # Issue #22898.
+        # Test that a RecursionError is raised when tstate->recursion_depth is
+        # equal to recursion_limit in PyErr_NormalizeException() and check
+        # that a ResourceWarning is printed.
+        # Prior to #22898, the recursivity of PyErr_NormalizeException() was
+        # controlled by tstate->recursion_depth and a PyExc_RecursionErrorInst
+        # singleton was being used in that case, that held traceback data and
+        # locals indefinitely and would cause a segfault in _PyExc_Fini() upon
+        # finalization of these locals.
+        code = """if 1:
+            import sys
+            from _testcapi import get_recursion_depth
+
+            class MyException(Exception): pass
+
+            def setrecursionlimit(depth):
+                while 1:
+                    try:
+                        sys.setrecursionlimit(depth)
+                        return depth
+                    except RecursionError:
+                        # sys.setrecursionlimit() raises a RecursionError if
+                        # the new recursion limit is too low (issue #25274).
+                        depth += 1
+
+            def recurse(cnt):
+                cnt -= 1
+                if cnt:
+                    recurse(cnt)
+                else:
+                    generator.throw(MyException)
+
+            def gen():
+                f = open(%a, mode='rb', buffering=0)
+                yield
+
+            generator = gen()
+            next(generator)
+            recursionlimit = sys.getrecursionlimit()
+            depth = get_recursion_depth()
+            try:
+                # Upon the last recursive invocation of recurse(),
+                # tstate->recursion_depth is equal to (recursion_limit - 1)
+                # and is equal to recursion_limit when _gen_throw() calls
+                # PyErr_NormalizeException().
+                recurse(setrecursionlimit(depth + 2) - depth - 1)
+            finally:
+                sys.setrecursionlimit(recursionlimit)
+                print('Done.')
+        """ % __file__
+        rc, out, err = script_helper.assert_python_failure("-Wd", "-c", code)
+        # Check that the program does not fail with SIGABRT.
+        self.assertEqual(rc, 1)
+        self.assertIn(b'RecursionError', err)
+        self.assertIn(b'ResourceWarning', err)
+        self.assertIn(b'Done.', out)
+
+    @cpython_only
+    def test_recursion_normalizing_infinite_exception(self):
+        # Issue #30697. Test that a RecursionError is raised when
+        # PyErr_NormalizeException() maximum recursion depth has been
+        # exceeded.
+        code = """if 1:
+            import _testcapi
+            try:
+                raise _testcapi.RecursingInfinitelyError
+            finally:
+                print('Done.')
+        """
+        rc, out, err = script_helper.assert_python_failure("-c", code)
+        self.assertEqual(rc, 1)
+        self.assertIn(b'RecursionError: maximum recursion depth exceeded '
+                      b'while normalizing an exception', err)
+        self.assertIn(b'Done.', out)
+
+    @cpython_only
+    def test_recursion_normalizing_with_no_memory(self):
+        # Issue #30697. Test that in the abort that occurs when there is no
+        # memory left and the size of the Python frames stack is greater than
+        # the size of the list of preallocated MemoryError instances, the
+        # Fatal Python error message mentions MemoryError.
+        code = """if 1:
+            import _testcapi
+            class C(): pass
+            def recurse(cnt):
+                cnt -= 1
+                if cnt:
+                    recurse(cnt)
+                else:
+                    _testcapi.set_nomemory(0)
+                    C()
+            recurse(16)
+        """
+        with SuppressCrashReport():
+            rc, out, err = script_helper.assert_python_failure("-c", code)
+            self.assertIn(b'Fatal Python error: Cannot recover from '
+                          b'MemoryErrors while normalizing exceptions.', err)
+
+    @cpython_only
+    def test_MemoryError(self):
+        # PyErr_NoMemory always raises the same exception instance.
+        # Check that the traceback is not doubled.
+        import traceback
+        from _testcapi import raise_memoryerror
+        def raiseMemError():
+            try:
+                raise_memoryerror()
+            except MemoryError as e:
+                tb = e.__traceback__
+            else:
+                self.fail("Should have raises a MemoryError")
+            return traceback.format_tb(tb)
+
+        tb1 = raiseMemError()
+        tb2 = raiseMemError()
+        self.assertEqual(tb1, tb2)
+
+    @cpython_only
+    def test_exception_with_doc(self):
+        import _testcapi
+        doc2 = "This is a test docstring."
+        doc4 = "This is another test docstring."
+
+        self.assertRaises(SystemError, _testcapi.make_exception_with_doc,
+                          "error1")
+
+        # test basic usage of PyErr_NewException
+        error1 = _testcapi.make_exception_with_doc("_testcapi.error1")
+        self.assertIs(type(error1), type)
+        self.assertTrue(issubclass(error1, Exception))
+        self.assertIsNone(error1.__doc__)
+
+        # test with given docstring
+        error2 = _testcapi.make_exception_with_doc("_testcapi.error2", doc2)
+        self.assertEqual(error2.__doc__, doc2)
+
+        # test with explicit base (without docstring)
+        error3 = _testcapi.make_exception_with_doc("_testcapi.error3",
+                                                   base=error2)
+        self.assertTrue(issubclass(error3, error2))
+
+        # test with explicit base tuple
+        class C(object):
+            pass
+        error4 = _testcapi.make_exception_with_doc("_testcapi.error4", doc4,
+                                                   (error3, C))
+        self.assertTrue(issubclass(error4, error3))
+        self.assertTrue(issubclass(error4, C))
+        self.assertEqual(error4.__doc__, doc4)
+
+        # test with explicit dictionary
+        error5 = _testcapi.make_exception_with_doc("_testcapi.error5", "",
+                                                   error4, {'a': 1})
+        self.assertTrue(issubclass(error5, error4))
+        self.assertEqual(error5.a, 1)
+        self.assertEqual(error5.__doc__, "")
+
+    @cpython_only
+    def test_memory_error_cleanup(self):
+        # Issue #5437: preallocated MemoryError instances should not keep
+        # traceback objects alive.
+        from _testcapi import raise_memoryerror
+        class C:
+            pass
+        wr = None
+        def inner():
+            nonlocal wr
+            c = C()
+            wr = weakref.ref(c)
+            raise_memoryerror()
+        # We cannot use assertRaises since it manually deletes the traceback
+        try:
+            inner()
+        except MemoryError as e:
+            self.assertNotEqual(wr(), None)
+        else:
+            self.fail("MemoryError not raised")
+        self.assertEqual(wr(), None)
+
+    @no_tracing
+    def test_recursion_error_cleanup(self):
+        # Same test as above, but with "recursion exceeded" errors
+        class C:
+            pass
+        wr = None
+        def inner():
+            nonlocal wr
+            c = C()
+            wr = weakref.ref(c)
+            inner()
+        # We cannot use assertRaises since it manually deletes the traceback
+        try:
+            inner()
+        except RecursionError as e:
+            self.assertNotEqual(wr(), None)
+        else:
+            self.fail("RecursionError not raised")
+        self.assertEqual(wr(), None)
+
+    def test_errno_ENOTDIR(self):
+        # Issue #12802: "not a directory" errors are ENOTDIR even on Windows
+        with self.assertRaises(OSError) as cm:
+            os.listdir(__file__)
+        self.assertEqual(cm.exception.errno, errno.ENOTDIR, cm.exception)
+
+    def test_unraisable(self):
+        # Issue #22836: PyErr_WriteUnraisable() should give sensible reports
+        class BrokenDel:
+            def __del__(self):
+                exc = ValueError("del is broken")
+                # The following line is included in the traceback report:
+                raise exc
+
+        class BrokenExceptionDel:
+            def __del__(self):
+                exc = BrokenStrException()
+                # The following line is included in the traceback report:
+                raise exc
+
+        for test_class in (BrokenDel, BrokenExceptionDel):
+            with self.subTest(test_class):
+                obj = test_class()
+                with captured_stderr() as stderr:
+                    del obj
+                report = stderr.getvalue()
+                self.assertIn("Exception ignored", report)
+                self.assertIn(test_class.__del__.__qualname__, report)
+                self.assertIn("test_exceptions.py", report)
+                self.assertIn("raise exc", report)
+                if test_class is BrokenExceptionDel:
+                    self.assertIn("BrokenStrException", report)
+                    if sys.version_info >= (3, 6):
+                        self.assertIn("", report)
+                else:
+                    self.assertIn("ValueError", report)
+                    self.assertIn("del is broken", report)
+                if sys.version_info >= (3, 6):
+                    self.assertTrue(report.endswith("\n"))
+
+    def test_unhandled(self):
+        # Check for sensible reporting of unhandled exceptions
+        for exc_type in (ValueError, BrokenStrException):
+            with self.subTest(exc_type):
+                try:
+                    exc = exc_type("test message")
+                    # The following line is included in the traceback report:
+                    raise exc
+                except exc_type:
+                    with captured_stderr() as stderr:
+                        sys.__excepthook__(*sys.exc_info())
+                report = stderr.getvalue()
+                self.assertIn("test_exceptions.py", report)
+                self.assertIn("raise exc", report)
+                self.assertIn(exc_type.__name__, report)
+                if exc_type is BrokenStrException:
+                    if sys.version_info >= (3, 6):
+                        self.assertIn("", report)
+                else:
+                    self.assertIn("test message", report)
+                if sys.version_info >= (3, 6):
+                    self.assertTrue(report.endswith("\n"))
+
+    @cpython_only
+    def test_memory_error_in_PyErr_PrintEx(self):
+        code = """if 1:
+            import _testcapi
+            class C(): pass
+            _testcapi.set_nomemory(0, %d)
+            C()
+        """
+
+        # Issue #30817: Abort in PyErr_PrintEx() when no memory.
+        # Span a large range of tests as the CPython code always evolves with
+        # changes that add or remove memory allocations.
+        for i in range(1, 20):
+            rc, out, err = script_helper.assert_python_failure("-c", code % i)
+            self.assertIn(rc, (1, 120))
+            self.assertIn(b'MemoryError', err)
+
+    def test_yield_in_nested_try_excepts(self):
+        #Issue #25612
+        class MainError(Exception):
+            pass
+
+        class SubError(Exception):
+            pass
+
+        def main():
+            try:
+                raise MainError()
+            except MainError:
+                try:
+                    yield
+                except SubError:
+                    pass
+                raise
+
+        coro = main()
+        coro.send(None)
+        with self.assertRaises(MainError):
+            coro.throw(SubError())
+
+    @unittest.skip('currently fails')  #  FIXME: fails in the "inside" assertion but not "outside"
+    def test_generator_doesnt_retain_old_exc2(self):
+        #Issue 28884#msg282532
+        def g():
+            try:
+                raise ValueError
+            except ValueError:
+                yield 1
+            self.assertEqual(sys.exc_info(), (None, None, None))  # inside
+            yield 2
+
+        gen = g()
+
+        try:
+            raise IndexError
+        except IndexError:
+            self.assertEqual(next(gen), 1)
+        self.assertEqual(sys.exc_info(), (None, None, None))  # outside
+        self.assertEqual(next(gen), 2)
+
+    def test_raise_in_generator(self):
+        #Issue 25612#msg304117
+        def g():
+            yield 1
+            raise
+            yield 2
+
+        with self.assertRaises(ZeroDivisionError):
+            i = g()
+            ZERO = 0
+            try:
+                1/ZERO
+            except:
+                next(i)
+                next(i)
+
+
+class ImportErrorTests(unittest.TestCase):
+
+    @unittest.skipIf(sys.version_info < (3, 6), "Requires Py3.6+")
+    def test_attributes(self):
+        # Setting 'name' and 'path' should not be a problem.
+        exc = ImportError('test')
+        self.assertIsNone(exc.name)
+        self.assertIsNone(exc.path)
+
+        exc = ImportError('test', name='somemodule')
+        self.assertEqual(exc.name, 'somemodule')
+        self.assertIsNone(exc.path)
+
+        exc = ImportError('test', path='somepath')
+        self.assertEqual(exc.path, 'somepath')
+        self.assertIsNone(exc.name)
+
+        exc = ImportError('test', path='somepath', name='somename')
+        self.assertEqual(exc.name, 'somename')
+        self.assertEqual(exc.path, 'somepath')
+
+        msg = ("'invalid' is an invalid keyword argument for ImportError"
+               if sys.version_info >= (3, 7) else ".*keyword argument.*")
+        with self.assertRaisesRegex(TypeError, msg):
+            ImportError('test', invalid='keyword')
+
+        with self.assertRaisesRegex(TypeError, msg):
+            ImportError('test', name='name', invalid='keyword')
+
+        with self.assertRaisesRegex(TypeError, msg):
+            ImportError('test', path='path', invalid='keyword')
+
+        with self.assertRaisesRegex(TypeError, msg):
+            ImportError(invalid='keyword')
+
+        with self.assertRaisesRegex(TypeError, msg):
+            ImportError('test', invalid='keyword', another=True)
+
+    @unittest.skipIf(sys.version_info < (3, 7), "requires Py3.7+")
+    def test_reset_attributes(self):
+        exc = ImportError('test', name='name', path='path')
+        self.assertEqual(exc.args, ('test',))
+        self.assertEqual(exc.msg, 'test')
+        self.assertEqual(exc.name, 'name')
+        self.assertEqual(exc.path, 'path')
+
+        # Reset not specified attributes
+        exc.__init__()
+        self.assertEqual(exc.args, ())
+        self.assertEqual(exc.msg, None)
+        self.assertEqual(exc.name, None)
+        self.assertEqual(exc.path, None)
+
+    def test_non_str_argument(self):
+        # Issue #15778
+        with check_warnings(('', BytesWarning), quiet=True):
+            arg = b'abc'
+            exc = ImportError(arg)
+            self.assertEqual(str(arg), str(exc))
+
+    @unittest.skipIf(sys.version_info < (3, 6), "Requires Py3.6+")
+    def test_copy_pickle(self):
+        for kwargs in (dict(),
+                       dict(name='somename'),
+                       dict(path='somepath'),
+                       dict(name='somename', path='somepath')):
+            orig = ImportError('test', **kwargs)
+            for proto in range(pickle.HIGHEST_PROTOCOL + 1):
+                exc = pickle.loads(pickle.dumps(orig, proto))
+                self.assertEqual(exc.args, ('test',))
+                self.assertEqual(exc.msg, 'test')
+                self.assertEqual(exc.name, orig.name)
+                self.assertEqual(exc.path, orig.path)
+            for c in copy.copy, copy.deepcopy:
+                exc = c(orig)
+                self.assertEqual(exc.args, ('test',))
+                self.assertEqual(exc.msg, 'test')
+                self.assertEqual(exc.name, orig.name)
+                self.assertEqual(exc.path, orig.path)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_fstring.pyx cython-0.20.1+1~202203241016-9537/tests/run/test_fstring.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/test_fstring.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_fstring.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -3,18 +3,21 @@
 # tag: allow_unknown_names, f_strings, pep498
 
 import ast
+import os
 import types
 import decimal
 import unittest
-import contextlib
 
 import sys
 IS_PY2 = sys.version_info[0] < 3
-IS_PY26 = sys.version_info[:2] < (2, 7)
+if IS_PY2:
+    # Define `ascii` as `repr` for Python2 as it functions
+    # the same way in that version.
+    ascii = repr
 
 from Cython.Build.Inline import cython_inline
 from Cython.TestUtils import CythonTest
-from Cython.Compiler.Errors import CompileError, hold_errors, release_errors, error_stack
+from Cython.Compiler.Errors import CompileError, hold_errors, init_thread, held_errors
 
 def cy_eval(s, **kwargs):
     return cython_inline('return ' + s, force=True, **kwargs)
@@ -31,55 +34,38 @@
 class TestCase(CythonTest):
     def assertAllRaise(self, exception_type, regex, error_strings):
         for str in error_strings:
+            hold_errors()
             if exception_type is SyntaxError:
                 try:
                     self.fragment(str)
                 except CompileError:
                     assert True
                 else:
-                    assert False, "Invalid Cython code failed to raise SyntaxError: %s" % str
+                    assert held_errors(), "Invalid Cython code failed to raise SyntaxError: %r" % str
+                finally:
+                    init_thread()  # reset error status
             else:
-                hold_errors()
                 try:
                     cython_inline(str, quiet=True)
                 except exception_type:
                     assert True
                 else:
-                    assert False, "Invalid Cython code failed to raise %s: %s" % (exception_type, str)
+                    assert False, "Invalid Cython code failed to raise %s: %r" % (exception_type, str)
                 finally:
-                    if error_stack:
-                        release_errors(ignore=True)
+                    init_thread()  # reset error status
 
     if IS_PY2:
         def assertEqual(self, first, second, msg=None):
             # strip u'' string prefixes in Py2
             if first != second and isinstance(first, unicode):
-                stripped_first = first.replace("u'", "'").replace('u"', '"')
+                stripped_first = first.replace(u"u'", u"'").replace(u'u"', u'"')
                 if stripped_first == second:
-                    first = stripped_first
-                elif stripped_first.decode('unicode_escape') == second:
-                    first = stripped_first.decode('unicode_escape')
+                    first = second
+                elif u'\\' in stripped_first and stripped_first.encode('utf8').decode('unicode_escape') == second:
+                    first = second
             super(TestCase, self).assertEqual(first, second, msg)
 
-        if IS_PY26:
-            @contextlib.contextmanager
-            def assertRaises(self, exc):
-                try:
-                    yield
-                except exc:
-                    pass
-                else:
-                    assert False, "exception '%s' not raised" % exc
-
-            def assertIn(self, value, collection):
-                self.assertTrue(value in collection)
-
     def test__format__lookup(self):
-        if IS_PY26:
-            return
-        elif IS_PY2:
-            raise unittest.SkipTest("Py3-only")
-
         # Make sure __format__ is looked up on the type, not the instance.
         class X:
             def __format__(self, spec):
@@ -107,7 +93,7 @@
         self.assertEqual(type(y).__format__(y, ''), 'class')
 
     def __test_ast(self):
-        # Inspired by http://bugs.python.org/issue24975
+        # Inspired by https://bugs.python.org/issue24975
         class X:
             def __init__(self):
                 self.called = False
@@ -130,18 +116,267 @@
         # Make sure x was called.
         self.assertTrue(x.called)
 
-    def __test_literal_eval(self):
-        # With no expressions, an f-string is okay.
-        self.assertEqual(ast.literal_eval("f'x'"), 'x')
-        self.assertEqual(ast.literal_eval("f'x' 'y'"), 'xy')
+    def __test_ast_line_numbers(self):
+        expr = """
+a = 10
+f'{a * x()}'"""
+        t = ast.parse(expr)
+        self.assertEqual(type(t), ast.Module)
+        self.assertEqual(len(t.body), 2)
+        # check `a = 10`
+        self.assertEqual(type(t.body[0]), ast.Assign)
+        self.assertEqual(t.body[0].lineno, 2)
+        # check `f'...'`
+        self.assertEqual(type(t.body[1]), ast.Expr)
+        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+        self.assertEqual(len(t.body[1].value.values), 1)
+        self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        # check the binop location
+        binop = t.body[1].value.values[0].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.Call)
+        self.assertEqual(binop.lineno, 3)
+        self.assertEqual(binop.left.lineno, 3)
+        self.assertEqual(binop.right.lineno, 3)
+        self.assertEqual(binop.col_offset, 3)
+        self.assertEqual(binop.left.col_offset, 3)
+        self.assertEqual(binop.right.col_offset, 7)
 
-        # But this should raise an error.
-        with self.assertRaisesRegex(ValueError, 'malformed node or string'):
-            ast.literal_eval("f'x{3}'")
+    def __test_ast_line_numbers_multiple_formattedvalues(self):
+        expr = """
+f'no formatted values'
+f'eggs {a * x()} spam {b + y()}'"""
+        t = ast.parse(expr)
+        self.assertEqual(type(t), ast.Module)
+        self.assertEqual(len(t.body), 2)
+        # check `f'no formatted value'`
+        self.assertEqual(type(t.body[0]), ast.Expr)
+        self.assertEqual(type(t.body[0].value), ast.JoinedStr)
+        self.assertEqual(t.body[0].lineno, 2)
+        # check `f'...'`
+        self.assertEqual(type(t.body[1]), ast.Expr)
+        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+        self.assertEqual(len(t.body[1].value.values), 4)
+        self.assertEqual(type(t.body[1].value.values[0]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[0].value), str)
+        self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
+        self.assertEqual(type(t.body[1].value.values[2]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[2].value), str)
+        self.assertEqual(type(t.body[1].value.values[3]), ast.FormattedValue)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        self.assertEqual(t.body[1].value.values[1].lineno, 3)
+        self.assertEqual(t.body[1].value.values[2].lineno, 3)
+        self.assertEqual(t.body[1].value.values[3].lineno, 3)
+        # check the first binop location
+        binop1 = t.body[1].value.values[1].value
+        self.assertEqual(type(binop1), ast.BinOp)
+        self.assertEqual(type(binop1.left), ast.Name)
+        self.assertEqual(type(binop1.op), ast.Mult)
+        self.assertEqual(type(binop1.right), ast.Call)
+        self.assertEqual(binop1.lineno, 3)
+        self.assertEqual(binop1.left.lineno, 3)
+        self.assertEqual(binop1.right.lineno, 3)
+        self.assertEqual(binop1.col_offset, 8)
+        self.assertEqual(binop1.left.col_offset, 8)
+        self.assertEqual(binop1.right.col_offset, 12)
+        # check the second binop location
+        binop2 = t.body[1].value.values[3].value
+        self.assertEqual(type(binop2), ast.BinOp)
+        self.assertEqual(type(binop2.left), ast.Name)
+        self.assertEqual(type(binop2.op), ast.Add)
+        self.assertEqual(type(binop2.right), ast.Call)
+        self.assertEqual(binop2.lineno, 3)
+        self.assertEqual(binop2.left.lineno, 3)
+        self.assertEqual(binop2.right.lineno, 3)
+        self.assertEqual(binop2.col_offset, 23)
+        self.assertEqual(binop2.left.col_offset, 23)
+        self.assertEqual(binop2.right.col_offset, 27)
 
-        # As should this, which uses a different ast node
+    def __test_ast_line_numbers_nested(self):
+        expr = """
+a = 10
+f'{a * f"-{x()}-"}'"""
+        t = ast.parse(expr)
+        self.assertEqual(type(t), ast.Module)
+        self.assertEqual(len(t.body), 2)
+        # check `a = 10`
+        self.assertEqual(type(t.body[0]), ast.Assign)
+        self.assertEqual(t.body[0].lineno, 2)
+        # check `f'...'`
+        self.assertEqual(type(t.body[1]), ast.Expr)
+        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+        self.assertEqual(len(t.body[1].value.values), 1)
+        self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        # check the binop location
+        binop = t.body[1].value.values[0].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.JoinedStr)
+        self.assertEqual(binop.lineno, 3)
+        self.assertEqual(binop.left.lineno, 3)
+        self.assertEqual(binop.right.lineno, 3)
+        self.assertEqual(binop.col_offset, 3)
+        self.assertEqual(binop.left.col_offset, 3)
+        self.assertEqual(binop.right.col_offset, 7)
+        # check the nested call location
+        self.assertEqual(len(binop.right.values), 3)
+        self.assertEqual(type(binop.right.values[0]), ast.Constant)
+        self.assertEqual(type(binop.right.values[0].value), str)
+        self.assertEqual(type(binop.right.values[1]), ast.FormattedValue)
+        self.assertEqual(type(binop.right.values[2]), ast.Constant)
+        self.assertEqual(type(binop.right.values[2].value), str)
+        self.assertEqual(binop.right.values[0].lineno, 3)
+        self.assertEqual(binop.right.values[1].lineno, 3)
+        self.assertEqual(binop.right.values[2].lineno, 3)
+        call = binop.right.values[1].value
+        self.assertEqual(type(call), ast.Call)
+        self.assertEqual(call.lineno, 3)
+        self.assertEqual(call.col_offset, 11)
+
+    def __test_ast_line_numbers_duplicate_expression(self):
+        """Duplicate expression
+
+        NOTE: this is currently broken, always sets location of the first
+        expression.
+        """
+        expr = """
+a = 10
+f'{a * x()} {a * x()} {a * x()}'
+"""
+        t = ast.parse(expr)
+        self.assertEqual(type(t), ast.Module)
+        self.assertEqual(len(t.body), 2)
+        # check `a = 10`
+        self.assertEqual(type(t.body[0]), ast.Assign)
+        self.assertEqual(t.body[0].lineno, 2)
+        # check `f'...'`
+        self.assertEqual(type(t.body[1]), ast.Expr)
+        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+        self.assertEqual(len(t.body[1].value.values), 5)
+        self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue)
+        self.assertEqual(type(t.body[1].value.values[1]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[1].value), str)
+        self.assertEqual(type(t.body[1].value.values[2]), ast.FormattedValue)
+        self.assertEqual(type(t.body[1].value.values[3]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[3].value), str)
+        self.assertEqual(type(t.body[1].value.values[4]), ast.FormattedValue)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        self.assertEqual(t.body[1].value.values[1].lineno, 3)
+        self.assertEqual(t.body[1].value.values[2].lineno, 3)
+        self.assertEqual(t.body[1].value.values[3].lineno, 3)
+        self.assertEqual(t.body[1].value.values[4].lineno, 3)
+        # check the first binop location
+        binop = t.body[1].value.values[0].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.Call)
+        self.assertEqual(binop.lineno, 3)
+        self.assertEqual(binop.left.lineno, 3)
+        self.assertEqual(binop.right.lineno, 3)
+        self.assertEqual(binop.col_offset, 3)
+        self.assertEqual(binop.left.col_offset, 3)
+        self.assertEqual(binop.right.col_offset, 7)
+        # check the second binop location
+        binop = t.body[1].value.values[2].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.Call)
+        self.assertEqual(binop.lineno, 3)
+        self.assertEqual(binop.left.lineno, 3)
+        self.assertEqual(binop.right.lineno, 3)
+        self.assertEqual(binop.col_offset, 3)  # FIXME: this is wrong
+        self.assertEqual(binop.left.col_offset, 3)  # FIXME: this is wrong
+        self.assertEqual(binop.right.col_offset, 7)  # FIXME: this is wrong
+        # check the third binop location
+        binop = t.body[1].value.values[4].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.Call)
+        self.assertEqual(binop.lineno, 3)
+        self.assertEqual(binop.left.lineno, 3)
+        self.assertEqual(binop.right.lineno, 3)
+        self.assertEqual(binop.col_offset, 3)  # FIXME: this is wrong
+        self.assertEqual(binop.left.col_offset, 3)  # FIXME: this is wrong
+        self.assertEqual(binop.right.col_offset, 7)  # FIXME: this is wrong
+
+    def __test_ast_line_numbers_multiline_fstring(self):
+        # See bpo-30465 for details.
+        expr = """
+a = 10
+f'''
+  {a
+     *
+       x()}
+non-important content
+'''
+"""
+        t = ast.parse(expr)
+        self.assertEqual(type(t), ast.Module)
+        self.assertEqual(len(t.body), 2)
+        # check `a = 10`
+        self.assertEqual(type(t.body[0]), ast.Assign)
+        self.assertEqual(t.body[0].lineno, 2)
+        # check `f'...'`
+        self.assertEqual(type(t.body[1]), ast.Expr)
+        self.assertEqual(type(t.body[1].value), ast.JoinedStr)
+        self.assertEqual(len(t.body[1].value.values), 3)
+        self.assertEqual(type(t.body[1].value.values[0]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[0].value), str)
+        self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue)
+        self.assertEqual(type(t.body[1].value.values[2]), ast.Constant)
+        self.assertEqual(type(t.body[1].value.values[2].value), str)
+        self.assertEqual(t.body[1].lineno, 3)
+        self.assertEqual(t.body[1].value.lineno, 3)
+        self.assertEqual(t.body[1].value.values[0].lineno, 3)
+        self.assertEqual(t.body[1].value.values[1].lineno, 3)
+        self.assertEqual(t.body[1].value.values[2].lineno, 3)
+        self.assertEqual(t.body[1].col_offset, 0)
+        self.assertEqual(t.body[1].value.col_offset, 0)
+        self.assertEqual(t.body[1].value.values[0].col_offset, 0)
+        self.assertEqual(t.body[1].value.values[1].col_offset, 0)
+        self.assertEqual(t.body[1].value.values[2].col_offset, 0)
+        # NOTE: the following lineno information and col_offset is correct for
+        # expressions within FormattedValues.
+        binop = t.body[1].value.values[1].value
+        self.assertEqual(type(binop), ast.BinOp)
+        self.assertEqual(type(binop.left), ast.Name)
+        self.assertEqual(type(binop.op), ast.Mult)
+        self.assertEqual(type(binop.right), ast.Call)
+        self.assertEqual(binop.lineno, 4)
+        self.assertEqual(binop.left.lineno, 4)
+        self.assertEqual(binop.right.lineno, 6)
+        self.assertEqual(binop.col_offset, 4)
+        self.assertEqual(binop.left.col_offset, 4)
+        self.assertEqual(binop.right.col_offset, 7)
+
+    def test_docstring(self):
+        def f():
+            f'''Not a docstring'''
+        self.assertIsNone(f.__doc__)
+        def g():
+            '''Not a docstring''' \
+            f''
+        self.assertIsNone(g.__doc__)
+
+    def __test_literal_eval(self):
         with self.assertRaisesRegex(ValueError, 'malformed node or string'):
-            ast.literal_eval("f'{3}'")
+            ast.literal_eval("f'x'")
 
     def __test_ast_compile_time_concat(self):
         x = ['']
@@ -152,34 +387,17 @@
         exec(c)
         self.assertEqual(x[0], 'foo3')
 
+    def test_compile_time_concat_errors(self):
+        self.assertAllRaise(SyntaxError,
+                            'cannot mix bytes and nonbytes literals',
+                            [r"""f'' b''""",
+                             r"""b'' f''""",
+                             ])
+
     def test_literal(self):
         self.assertEqual(f'', '')
         self.assertEqual(f'a', 'a')
         self.assertEqual(f' ', ' ')
-        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}',
-                         '\N{GREEK CAPITAL LETTER DELTA}')
-        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}',
-                         '\u0394')
-        self.assertEqual(f'\N{True}', '\u22a8')
-        self.assertEqual(rf'\N{True}', r'\NTrue')
-
-    def test_escape_order(self):
-        # note that hex(ord('{')) == 0x7b, so this
-        #  string becomes f'a{4*10}b'
-        self.assertEqual(f'a\u007b4*10}b', 'a40b')
-        self.assertEqual(f'a\x7b4*10}b', 'a40b')
-        self.assertEqual(f'a\x7b4*10\N{RIGHT CURLY BRACKET}b', 'a40b')
-        self.assertEqual(f'{"a"!\N{LATIN SMALL LETTER R}}', "'a'")
-        self.assertEqual(f'{10\x3a02X}', '0A')
-        self.assertEqual(f'{10:02\N{LATIN CAPITAL LETTER X}}', '0A')
-
-        self.assertAllRaise(SyntaxError, "f-string: single '}' is not allowed",
-                            [r"""f'a{\u007b4*10}b'""",    # mis-matched brackets
-                             ])
-        self.assertAllRaise(SyntaxError, 'unexpected character after line continuation character',
-                            [r"""f'{"a"\!r}'""",
-                             r"""f'{a\!r}'""",
-                             ])
 
     def test_unterminated_string(self):
         self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
@@ -190,9 +408,27 @@
                              ])
 
     def test_mismatched_parens(self):
-        self.assertAllRaise(SyntaxError, 'f-string: mismatched',
+        self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' "
+                            r"does not match opening parenthesis '\('",
                             ["f'{((}'",
                              ])
+        self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\)' "
+                            r"does not match opening parenthesis '\['",
+                            ["f'{a[4)}'",
+                            ])
+        self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\]' "
+                            r"does not match opening parenthesis '\('",
+                            ["f'{a(4]}'",
+                            ])
+        self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' "
+                            r"does not match opening parenthesis '\['",
+                            ["f'{a[4}'",
+                            ])
+        self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' "
+                            r"does not match opening parenthesis '\('",
+                            ["f'{a(4}'",
+                            ])
+        self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'")
 
     def test_double_braces(self):
         self.assertEqual(f'{{', '{')
@@ -203,6 +439,14 @@
         self.assertEqual(f'a}}', 'a}')
         self.assertEqual(f'}}b', '}b')
         self.assertEqual(f'a}}b', 'a}b')
+        self.assertEqual(f'{{}}', '{}')
+        self.assertEqual(f'a{{}}', 'a{}')
+        self.assertEqual(f'{{b}}', '{b}')
+        self.assertEqual(f'{{}}c', '{}c')
+        self.assertEqual(f'a{{b}}', 'a{b}')
+        self.assertEqual(f'a{{}}c', 'a{}c')
+        self.assertEqual(f'{{b}}c', '{b}c')
+        self.assertEqual(f'a{{b}}c', 'a{b}c')
 
         self.assertEqual(f'{{{10}', '{10')
         self.assertEqual(f'}}{10}', '}10')
@@ -258,9 +502,14 @@
         self.assertEqual(f'{"#"}', '#')
         self.assertEqual(f'{d["#"]}', 'hash')
 
-        self.assertAllRaise(SyntaxError, "f-string cannot include '#'",
+        self.assertAllRaise(SyntaxError, "f-string expression part cannot include '#'",
                             ["f'{1#}'",   # error because the expression becomes "(1#)"
                              "f'{3(#)}'",
+                             "f'{#}'",
+                             ])
+        self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'",
+                            ["f'{)#}'",   # When wrapped in parens, this becomes
+                                          #  '()#)'.  Make sure that doesn't compile.
                              ])
 
     def test_many_expressions(self):
@@ -275,29 +524,35 @@
         width = 1
 
         # Test around 256.
-        for i in range(250, 260):
-            self.assertEqual(cy_eval(build_fstr(i), x=x, width=width), (x+' ')*i)
+        # for i in range(250, 260):
+        #     self.assertEqual(eval(build_fstr(i)), (x+' ')*i)
+        self.assertEqual(
+            cy_eval('[' + ', '.join(build_fstr(i) for i in range(250, 260)) + ']', x=x, width=width),
+            [(x+' ')*i for i in range(250, 260)],
+        )
 
         # Test concatenating 2 largs fstrings.
+        # self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256))
         self.assertEqual(cy_eval(build_fstr(255)*3, x=x, width=width), (x+' ')*(255*3))  # CPython uses 255*256
 
         s = build_fstr(253, '{x:{width}} ')
+        # self.assertEqual(eval(s), (x+' ')*254)
         self.assertEqual(cy_eval(s, x=x, width=width), (x+' ')*254)
 
         # Test lots of expressions and constants, concatenated.
         s = "f'{1}' 'x' 'y'" * 1024
+        # self.assertEqual(eval(s), '1xy' * 1024)
         self.assertEqual(cy_eval(s, x=x, width=width), '1xy' * 1024)
 
     def test_format_specifier_expressions(self):
         width = 10
         precision = 4
         value = decimal.Decimal('12.34567')
-        if not IS_PY26:
-            self.assertEqual(f'result: {value:{width}.{precision}}', 'result:      12.35')
-            self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result:      12.35')
-            self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result:      12.35')
-            self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result:      12.35')
-            self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result:      12.35')
         self.assertEqual(f'{10:#{1}0x}', '       0xa')
         self.assertEqual(f'{10:{"#"}1{0}{"x"}}', '       0xa')
         self.assertEqual(f'{-10:-{"#"}1{0}x}', '      -0xa')
@@ -310,14 +565,13 @@
                              # This looks like a nested format spec.
                              ])
 
-        self.assertAllRaise(SyntaxError, "invalid syntax",
-                            [# Invalid sytax inside a nested spec.
+        self.assertAllRaise(SyntaxError, "f-string: invalid syntax",
+                            [# Invalid syntax inside a nested spec.
                              "f'{4:{/5}}'",
                              ])
 
         # CYTHON: The nesting restriction seems rather arbitrary. Ignoring it for now and instead test that it works.
-        if not IS_PY26:
-            self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result:      12.35')
+        self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result:      12.35')
         #self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply",
         #                    [# Can't nest format specifiers.
         #                     "f'result: {value:{width:{0}}.{precision:1}}'",
@@ -349,8 +603,10 @@
                              "f'{ !r}'",
                              "f'{10:{ }}'",
                              "f' { } '",
-                             r"f'{\n}'",
-                             r"f'{\n \n}'",
+
+                             # The Python parser ignores also the following
+                             # whitespace characters in additional to a space.
+                             "f'''{\t\f\r\n}'''",
 
                              # Catch the empty expression before the
                              #  invalid conversion.
@@ -372,6 +628,12 @@
                              "f'{:x'",
                              ])
 
+        # Different error message is raised for other whitespace characters.
+        self.assertAllRaise(SyntaxError, r"invalid non-printable character U\+00A0",
+                            ["f'''{\xa0}'''",
+                             #"\xa0",
+                             ])
+
     def test_parens_in_expressions(self):
         self.assertEqual(f'{3,}', '(3,)')
 
@@ -379,12 +641,12 @@
         #  are added around it. But we shouldn't go from an invalid
         #  expression to a valid one. The added parens are just
         #  supposed to allow whitespace (including newlines).
-        self.assertAllRaise(SyntaxError, 'invalid syntax',
+        self.assertAllRaise(SyntaxError, 'f-string: invalid syntax',
                             ["f'{,}'",
                              "f'{,}'",  # this is (,), which is an error
                              ])
 
-        self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
+        self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'",
                             ["f'{3)+(4}'",
                              ])
 
@@ -392,24 +654,102 @@
                             ["f'{\n}'",
                              ])
 
+    def test_backslashes_in_string_part(self):
+        self.assertEqual(f'\t', '\t')
+        self.assertEqual(r'\t', '\\t')
+        self.assertEqual(rf'\t', '\\t')
+        self.assertEqual(f'{2}\t', '2\t')
+        self.assertEqual(f'{2}\t{3}', '2\t3')
+        self.assertEqual(f'\t{3}', '\t3')
+
+        self.assertEqual(f'\u0394', '\u0394')
+        self.assertEqual(r'\u0394', '\\u0394')
+        self.assertEqual(rf'\u0394', '\\u0394')
+        self.assertEqual(f'{2}\u0394', '2\u0394')
+        self.assertEqual(f'{2}\u0394{3}', '2\u03943')
+        self.assertEqual(f'\u0394{3}', '\u03943')
+
+        self.assertEqual(f'\U00000394', '\u0394')
+        self.assertEqual(r'\U00000394', '\\U00000394')
+        self.assertEqual(rf'\U00000394', '\\U00000394')
+        self.assertEqual(f'{2}\U00000394', '2\u0394')
+        self.assertEqual(f'{2}\U00000394{3}', '2\u03943')
+        self.assertEqual(f'\U00000394{3}', '\u03943')
+
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}', '\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'{2}\N{GREEK CAPITAL LETTER DELTA}{3}', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}{3}', '\u03943')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}', '2\u0394')
+        self.assertEqual(f'2\N{GREEK CAPITAL LETTER DELTA}3', '2\u03943')
+        self.assertEqual(f'\N{GREEK CAPITAL LETTER DELTA}3', '\u03943')
+
+        self.assertEqual(f'\x20', ' ')
+        self.assertEqual(r'\x20', '\\x20')
+        self.assertEqual(rf'\x20', '\\x20')
+        self.assertEqual(f'{2}\x20', '2 ')
+        self.assertEqual(f'{2}\x20{3}', '2 3')
+        self.assertEqual(f'\x20{3}', ' 3')
+
+        self.assertEqual(f'2\x20', '2 ')
+        self.assertEqual(f'2\x203', '2 3')
+        self.assertEqual(f'\x203', ' 3')
+
+        #with self.assertWarns(DeprecationWarning):  # invalid escape sequence
+        #    value = cy_eval(r"f'\{6*7}'")
+        #self.assertEqual(value, '\\42')
+        self.assertEqual(f'\\{6*7}', '\\42')
+        self.assertEqual(fr'\{6*7}', '\\42')
+
+        AMPERSAND = 'spam'
+        # Get the right unicode character (&), or pick up local variable
+        # depending on the number of backslashes.
+        self.assertEqual(f'\N{AMPERSAND}', '&')
+        self.assertEqual(f'\\N{AMPERSAND}', '\\Nspam')
+        self.assertEqual(fr'\N{AMPERSAND}', '\\Nspam')
+        self.assertEqual(f'\\\N{AMPERSAND}', '\\&')
+
+    def test_misformed_unicode_character_name(self):
+        # These test are needed because unicode names are parsed
+        # differently inside f-strings.
+        self.assertAllRaise(SyntaxError, r"\(unicode error\) 'unicodeescape' codec can't decode bytes in position .*: malformed \\N character escape",
+                            [r"f'\N'",
+                             r"f'\N{'",
+                             r"f'\N{GREEK CAPITAL LETTER DELTA'",
+
+                             # Here are the non-f-string versions,
+                             #  which should give the same errors.
+                             r"'\N'",
+                             r"'\N{'",
+                             r"'\N{GREEK CAPITAL LETTER DELTA'",
+                             ])
+
+    def test_no_backslashes_in_expression_part(self):
+        self.assertAllRaise(SyntaxError, 'f-string expression part cannot include a backslash',
+                            [r"f'{\'a\'}'",
+                             r"f'{\t3}'",
+                             r"f'{\}'",
+                             r"rf'{\'a\'}'",
+                             r"rf'{\t3}'",
+                             r"rf'{\}'",
+                             r"""rf'{"\N{LEFT CURLY BRACKET}"}'""",
+                             r"f'{\n}'",
+                             ])
+
+    def test_no_escapes_for_braces(self):
+        """
+        Only literal curly braces begin an expression.
+        """
+        # \x7b is '{'.
+        self.assertEqual(f'\x7b1+1}}', '{1+1}')
+        self.assertEqual(f'\x7b1+1', '{1+1')
+        self.assertEqual(f'\u007b1+1', '{1+1')
+        self.assertEqual(f'\N{LEFT CURLY BRACKET}1+1\N{RIGHT CURLY BRACKET}', '{1+1}')
+
     def test_newlines_in_expressions(self):
         self.assertEqual(f'{0}', '0')
-        self.assertEqual(f'{0\n}', '0')
-        self.assertEqual(f'{0\r}', '0')
-        self.assertEqual(f'{\n0\n}', '0')
-        self.assertEqual(f'{\r0\r}', '0')
-        self.assertEqual(f'{\n0\r}', '0')
-        self.assertEqual(f'{\n0}', '0')
-        self.assertEqual(f'{3+\n4}', '7')
-        self.assertEqual(f'{3+\\\n4}', '7')
         self.assertEqual(rf'''{3+
 4}''', '7')
-        self.assertEqual(f'''{3+\
-4}''', '7')
-
-        self.assertAllRaise(SyntaxError, 'f-string: empty expression not allowed',
-                            [r"f'{\n}'",
-                             ])
 
     def test_lambda(self):
         x = 5
@@ -420,7 +760,7 @@
 
         # lambda doesn't work without parens, because the colon
         #  makes the parser think it's a format_spec
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+        self.assertAllRaise(SyntaxError, 'f-string: invalid syntax',
                             ["f'{lambda x:x}'",
                              ])
 
@@ -429,9 +769,11 @@
         #  a function into a generator
         def fn(y):
             f'y:{yield y*2}'
+            f'{yield}'
 
         g = fn(4)
         self.assertEqual(next(g), 8)
+        self.assertEqual(next(g), None)
 
     def test_yield_send(self):
         def fn(x):
@@ -445,9 +787,6 @@
     def test_expressions_with_triple_quoted_strings(self):
         self.assertEqual(f"{'''x'''}", 'x')
         self.assertEqual(f"{'''eric's'''}", "eric's")
-        self.assertEqual(f'{"""eric\'s"""}', "eric's")
-        self.assertEqual(f"{'''eric\"s'''}", 'eric"s')
-        self.assertEqual(f'{"""eric"s"""}', 'eric"s')
 
         # Test concatenation within an expression
         self.assertEqual(f'{"x" """eric"s""" "y"}', 'xeric"sy')
@@ -549,14 +888,9 @@
         y = 5
         self.assertEqual(f'{f"{0}"*3}', '000')
         self.assertEqual(f'{f"{y}"*3}', '555')
-        self.assertEqual(f'{f"{\'x\'}"*3}', 'xxx')
-
-        self.assertEqual(f"{r'x' f'{\"s\"}'}", 'xs')
-        self.assertEqual(f"{r'x'rf'{\"s\"}'}", 'xs')
 
     def test_invalid_string_prefixes(self):
-        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
-                            ["fu''",
+        single_quote_cases = ["fu''",
                              "uf''",
                              "Fu''",
                              "fU''",
@@ -570,29 +904,29 @@
                              "ruf''",
                              "FUR''",
                              "Fur''",
-                             ])
+                             "fb''",
+                             "fB''",
+                             "Fb''",
+                             "FB''",
+                             "bf''",
+                             "bF''",
+                             "Bf''",
+                             "BF''",]
+        double_quote_cases = [case.replace("'", '"') for case in single_quote_cases]
+        self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing',
+                            single_quote_cases + double_quote_cases)
 
     def test_leading_trailing_spaces(self):
         self.assertEqual(f'{ 3}', '3')
         self.assertEqual(f'{  3}', '3')
-        self.assertEqual(f'{\t3}', '3')
-        self.assertEqual(f'{\t\t3}', '3')
         self.assertEqual(f'{3 }', '3')
         self.assertEqual(f'{3  }', '3')
-        self.assertEqual(f'{3\t}', '3')
-        self.assertEqual(f'{3\t\t}', '3')
 
         self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]}}',
                          'expr={1: 2}')
         self.assertEqual(f'expr={ {x: y for x, y in [(1, 2), ]} }',
                          'expr={1: 2}')
 
-    def test_character_name(self):
-        self.assertEqual(f'{4}\N{GREEK CAPITAL LETTER DELTA}{3}',
-                         '4\N{GREEK CAPITAL LETTER DELTA}3')
-        self.assertEqual(f'{{}}\N{GREEK CAPITAL LETTER DELTA}{3}',
-                         '{}\N{GREEK CAPITAL LETTER DELTA}3')
-
     def test_not_equal(self):
         # There's a special test for this because there's a special
         #  case in the f-string parser to look for != as not ending an
@@ -603,12 +937,17 @@
         self.assertEqual(f'{3!=4!s}', 'True')
         self.assertEqual(f'{3!=4!s:.3}', 'Tru')
 
+    def test_equal_equal(self):
+        # Because an expression ending in = has special meaning,
+        # there's a special test for ==. Make sure it works.
+
+        self.assertEqual(f'{0==1}', 'False')
+
     def test_conversions(self):
         self.assertEqual(f'{3.14:10.10}', '      3.14')
-        if not IS_PY26:
-            self.assertEqual(f'{3.14!s:10.10}', '3.14      ')
-            self.assertEqual(f'{3.14!r:10.10}', '3.14      ')
-            self.assertEqual(f'{3.14!a:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!s:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!r:10.10}', '3.14      ')
+        self.assertEqual(f'{3.14!a:10.10}', '3.14      ')
 
         self.assertEqual(f'{"a"}', 'a')
         self.assertEqual(f'{"a"!r}', "'a'")
@@ -620,20 +959,14 @@
         # Not a conversion, but show that ! is allowed in a format spec.
         self.assertEqual(f'{3.14:!<10.10}', '3.14!!!!!!')
 
-        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"}', '\u0394')
-        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!r}', "'\u0394'")
-        self.assertEqual(f'{"\N{GREEK CAPITAL LETTER DELTA}"!a}', "'\\u0394'")
-
         self.assertAllRaise(SyntaxError, 'f-string: invalid conversion character',
                             ["f'{3!g}'",
                              "f'{3!A}'",
-                             "f'{3!A}'",
-                             "f'{3!A}'",
+                             "f'{3!3}'",
+                             "f'{3!G}'",
                              "f'{3!!}'",
                              "f'{3!:}'",
-                             "f'{3!\N{GREEK CAPITAL LETTER DELTA}}'",
                              "f'{3! s}'",  # no space before conversion char
-                             "f'{x!\\x00:.<10}'",
                              ])
 
         self.assertAllRaise(SyntaxError, "f-string: expecting '}'",
@@ -663,10 +996,10 @@
                              "f'}'",
                              "f'x}'",
                              "f'x}x'",
+                             r"f'\u007b}'",
 
                              # Can't have { or } in a format spec.
                              "f'{3:}>10}'",
-                             r"f'{3:\\}>10}'",
                              "f'{3:}}>10}'",
                              ])
 
@@ -680,16 +1013,13 @@
                              "f'{3!s:3'",
                              "f'x{'",
                              "f'x{x'",
+                             "f'{x'",
                              "f'{3:s'",
                              "f'{{{'",
                              "f'{{}}{'",
                              "f'{'",
                              ])
 
-        self.assertAllRaise(SyntaxError, 'invalid syntax',
-                            [r"f'{3:\\{>10}'",
-                             ])
-
         # But these are just normal strings.
         self.assertEqual(f'{"{"}', '{')
         self.assertEqual(f'{"}"}', '}')
@@ -752,15 +1082,10 @@
         self.assertEqual('{d[a]}'.format(d=d), 'string')
         self.assertEqual('{d[0]}'.format(d=d), 'integer')
 
-    def test_invalid_expressions(self):
-        self.assertAllRaise(SyntaxError, 'invalid syntax',
-                            [r"f'{a[4)}'",
-                             r"f'{a(4]}'",
-                            ])
-
     def test_errors(self):
         # see issue 26287
-        self.assertAllRaise((TypeError, ValueError), 'non-empty',  # TypeError in Py3.4+
+        exc = ValueError if sys.version_info < (3, 4) else TypeError
+        self.assertAllRaise(exc, 'unsupported',
                             [r"f'{(lambda: 0):x}'",
                              r"f'{(0,):x}'",
                              ])
@@ -769,6 +1094,16 @@
                              r"f'{1000:j}'",
                             ])
 
+    def __test_filename_in_syntaxerror(self):
+        # see issue 38964
+        with temp_cwd() as cwd:
+            file_path = os.path.join(cwd, 't.py')
+            with open(file_path, 'w') as f:
+                f.write('f"{a b}"') # This generates a SyntaxError
+            _, _, stderr = assert_python_failure(file_path,
+                                                 PYTHONIOENCODING='ascii')
+        self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr)
+
     def test_loop(self):
         for i in range(1000):
             self.assertEqual(f'i:{i}', 'i:' + str(i))
@@ -778,33 +1113,139 @@
              "'": 'squote',
              'foo': 'bar',
              }
-        self.assertEqual(f'{d["\'"]}', 'squote')
-        self.assertEqual(f"{d['\"']}", 'dquote')
-
         self.assertEqual(f'''{d["'"]}''', 'squote')
         self.assertEqual(f"""{d['"']}""", 'dquote')
 
         self.assertEqual(f'{d["foo"]}', 'bar')
         self.assertEqual(f"{d['foo']}", 'bar')
-        self.assertEqual(f'{d[\'foo\']}', 'bar')
-        self.assertEqual(f"{d[\"foo\"]}", 'bar')
 
-    def test_escaped_quotes(self):
-        d = {'"': 'a',
-             "'": 'b'}
-
-        self.assertEqual(fr"{d['\"']}", 'a')
-        self.assertEqual(fr'{d["\'"]}', 'b')
-        self.assertEqual(fr"{'\"'}", '"')
-        self.assertEqual(fr'{"\'"}', "'")
-        self.assertEqual(f'{"\\"3"}', '"3')
-
-        self.assertAllRaise(SyntaxError, 'f-string: unterminated string',
-                            [r'''f'{"""\\}' ''',  # Backslash at end of expression
-                             ])
-        self.assertAllRaise(SyntaxError, 'unexpected character after line continuation',
-                            [r"rf'{3\}'",
-                             ])
+    def __test_backslash_char(self):
+        # Check eval of a backslash followed by a control char.
+        # See bpo-30682: this used to raise an assert in pydebug mode.
+        self.assertEqual(cy_eval('f"\\\n"'), '')
+        self.assertEqual(cy_eval('f"\\\r"'), '')
+
+    def test_debug_conversion(self):
+        x = 'A string'
+        self.assertEqual(f'{x=}', 'x=' + repr(x))
+        self.assertEqual(f'{x =}', 'x =' + repr(x))
+        self.assertEqual(f'{x=!s}', 'x=' + str(x))
+        self.assertEqual(f'{x=!r}', 'x=' + repr(x))
+        self.assertEqual(f'{x=!a}', 'x=' + ascii(x))
+
+        x = 2.71828
+        self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f'))
+        self.assertEqual(f'{x=:}', 'x=' + format(x, ''))
+        self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20'))
+        self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20'))
+        self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20'))
+
+        x = 9
+        self.assertEqual(f'{3*x+15=}', '3*x+15=42')
+
+        # There is code in ast.c that deals with non-ascii expression values.  So,
+        # use a unicode identifier to trigger that.
+        tenπ = 31.4
+        self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40')
+
+        # Also test with Unicode in non-identifiers.
+        if not IS_PY2:  # unicode representation looks different right now - not sure if that's a good thing
+            self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'')
+
+        # Make sure nested fstrings still work.
+        self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****')
+
+        # Make sure text before and after an expression with = works
+        # correctly.
+        pi = 'π'
+        if not IS_PY2:  # unicode representation looks different right now - not sure if that's a good thing
+            self.assertEqual(f'alpha α {pi=} ω omega', u"alpha α pi='π' ω omega")
+
+        # Check multi-line expressions.
+        self.assertEqual(f'''{
+3
+=}''', '\n3\n=3')
+
+        # Since = is handled specially, make sure all existing uses of
+        # it still work.
+
+        self.assertEqual(f'{0==1}', 'False')
+        self.assertEqual(f'{0!=1}', 'True')
+        self.assertEqual(f'{0<=1}', 'True')
+        self.assertEqual(f'{0>=1}', 'False')
+        self.assertEqual(f'{(x:="5")}', '5')
+        self.assertEqual(x, '5')
+        self.assertEqual(f'{(x:=5)}', '5')
+        self.assertEqual(x, 5)
+        self.assertEqual(f'{"="}', '=')
+
+        x = 20
+        # This isn't an assignment expression, it's 'x', with a format
+        # spec of '=10'.  See test_walrus: you need to use parens.
+        self.assertEqual(f'{x:=10}', '        20')
+
+        # Test named function parameters, to make sure '=' parsing works
+        # there.
+        def f(a):
+            nonlocal x
+            oldx = x
+            x = a
+            return oldx
+        x = 0
+        self.assertEqual(f'{f(a="3=")}', '0')
+        self.assertEqual(x, '3=')
+        self.assertEqual(f'{f(a=4)}', '3=')
+        self.assertEqual(x, 4)
+
+        # Make sure __format__ is being called.
+        class C:
+            def __format__(self, s):
+                return f'FORMAT-{s}'
+            def __repr__(self):
+                return 'REPR'
+
+        self.assertEqual(f'{C()=}', 'C()=REPR')
+        self.assertEqual(f'{C()=!r}', 'C()=REPR')
+        self.assertEqual(f'{C()=:}', 'C()=FORMAT-')
+        self.assertEqual(f'{C()=: }', 'C()=FORMAT- ')
+        self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x')
+        self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********')
+
+        self.assertRaises(SyntaxError, eval, "f'{C=]'")
+
+        # Make sure leading and following text works.
+        x = 'foo'
+        self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y')
+
+        # Make sure whitespace around the = works.
+        self.assertEqual(f'X{x  =}Y', 'Xx  ='+repr(x)+'Y')
+        self.assertEqual(f'X{x=  }Y', 'Xx=  '+repr(x)+'Y')
+        self.assertEqual(f'X{x  =  }Y', 'Xx  =  '+repr(x)+'Y')
+
+        # These next lines contains tabs.  Backslash escapes don't
+        # work in f-strings.
+        # patchcheck doesn't like these tabs.  So the only way to test
+        # this will be to dynamically created and exec the f-strings.  But
+        # that's such a hassle I'll save it for another day.  For now, convert
+        # the tabs to spaces just to shut up patchcheck.
+        #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y')
+        #self.assertEqual(f'X{x =       }Y', 'Xx\t=\t'+repr(x)+'Y')
+
+
+    def test_walrus(self):
+        x = 20
+        # This isn't an assignment expression, it's 'x', with a format
+        # spec of '=10'.
+        self.assertEqual(f'{x:=10}', '        20')
+
+        # This is an assignment expression, which requires parens.
+        self.assertEqual(f'{(x:=10)}', '10')
+        self.assertEqual(x, 10)
+
+    def test_invalid_syntax_error_message(self):
+        # with self.assertRaisesRegex(SyntaxError, "f-string: invalid syntax"):
+        #     compile("f'{a $ b}'", "?", "exec")
+        self.assertAllRaise(CompileError, "f-string: invalid syntax", ["f'{a $ b}'"])
 
 
 if __name__ == '__main__':
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_genericclass_exttype.pyx cython-0.20.1+1~202203241016-9537/tests/run/test_genericclass_exttype.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/test_genericclass_exttype.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_genericclass_exttype.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,101 @@
+# mode: run
+# cython: language_level=3
+
+import unittest
+import sys
+
+
+cdef class UnSupport: pass
+
+cdef class Unpack:
+    para_list = []
+    def __class_getitem__(*args, **kwargs):
+        Unpack.para_list.extend([args, kwargs])
+
+cdef class Format:
+    def __class_getitem__(cls, item):
+        return f'{cls.__name__}[{item.__name__}]'
+
+cdef class ExFormat(Format): pass
+
+cdef class Override:
+    def __class_getitem__(cls, item):
+        return 'Should not see this'
+
+cdef class Covered(Override):
+    def __class_getitem__(cls, item):
+        return f'{cls.__name__}[{item.__name__}]'
+
+cdef class Decorated:
+    @classmethod
+    def __class_getitem__(cls, item):
+        return f'{cls.__name__}[{item.__name__}]'
+
+cdef class ExDecorated(Decorated): pass
+
+cdef class Invalid1:
+    def __class_getitem__(cls): pass
+
+cdef class Invalid2:
+    def __class_getitem__(cls, item1, item2): pass
+
+cdef class Invalid3:
+    cdef dict __dict__
+    def __init__(self):
+        self.__class_getitem__ = lambda cls, items: 'This will not work'
+
+cdef class Invalid4:
+    __class_getitem__ = "Surprise!"
+
+
+class TestClassGetitem(unittest.TestCase):
+    # BEGIN - Additional tests from cython
+    def test_no_class_getitem(self):
+        # PyPy<7.3.8 raises AttributeError on __class_getitem__
+        if hasattr(sys, "pypy_version_info")  and sys.pypy_version_info < (7, 3, 8):
+            err = AttributeError
+        else:
+            err = TypeError
+        with self.assertRaises(err):
+            UnSupport[int]
+
+    # END - Additional tests from cython
+
+    def test_class_getitem(self):
+        Unpack[int, str]
+        self.assertEqual(Unpack.para_list[0], (Unpack, (int, str)))
+        self.assertEqual(Unpack.para_list[1], {})
+
+    def test_class_getitem_format(self):
+        self.assertEqual(Format[int], 'Format[int]')
+        self.assertEqual(Format[Format], 'Format[Format]')
+
+    def test_class_getitem_inheritance(self):
+        self.assertEqual(ExFormat[int], 'ExFormat[int]')
+        self.assertEqual(ExFormat[ExFormat], 'ExFormat[ExFormat]')
+
+    def test_class_getitem_inheritance_2(self):
+        self.assertEqual(Covered[int], 'Covered[int]')
+        self.assertEqual(Covered[Covered], 'Covered[Covered]')
+
+    def test_class_getitem_classmethod(self):
+        self.assertEqual(ExDecorated[int], 'ExDecorated[int]')
+        self.assertEqual(ExDecorated[ExDecorated], 'ExDecorated[ExDecorated]')
+
+    def test_class_getitem_errors(self):
+        with self.assertRaises(TypeError):
+            Invalid1[int]
+        with self.assertRaises(TypeError):
+            Invalid2[int]
+
+    def test_class_getitem_errors_2(self):
+        with self.assertRaises(TypeError):
+            Format()[int]
+        with self.assertRaises(TypeError):
+            Invalid3()[int]
+        with self.assertRaises(TypeError):
+            Invalid4[int]
+
+
+if __name__ == '__main__':
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_genericclass.py cython-0.20.1+1~202203241016-9537/tests/run/test_genericclass.py
--- cython-0.20.1+1~201611251650-6686/tests/run/test_genericclass.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_genericclass.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,294 @@
+# mode: run
+# tag: pure3.7
+# cython: language_level=3
+
+# COPIED FROM CPython 3.7
+
+import contextlib
+import unittest
+import sys
+
+class TestMROEntry(unittest.TestCase):
+    def test_mro_entry_signature(self):
+        tested = []
+        class B: ...
+        class C:
+            def __mro_entries__(self, *args, **kwargs):
+                tested.extend([args, kwargs])
+                return (C,)
+        c = C()
+        self.assertEqual(tested, [])
+        class D(B, c): ...
+        self.assertEqual(tested[0], ((B, c),))
+        self.assertEqual(tested[1], {})
+
+    def test_mro_entry(self):
+        tested = []
+        class A: ...
+        class B: ...
+        class C:
+            def __mro_entries__(self, bases):
+                tested.append(bases)
+                return (self.__class__,)
+        c = C()
+        self.assertEqual(tested, [])
+        class D(A, c, B): ...
+        self.assertEqual(tested[-1], (A, c, B))
+        self.assertEqual(D.__bases__, (A, C, B))
+        self.assertEqual(D.__orig_bases__, (A, c, B))
+        self.assertEqual(D.__mro__, (D, A, C, B, object))
+        d = D()
+        class E(d): ...
+        self.assertEqual(tested[-1], (d,))
+        self.assertEqual(E.__bases__, (D,))
+
+    def test_mro_entry_none(self):
+        tested = []
+        class A: ...
+        class B: ...
+        class C:
+            def __mro_entries__(self, bases):
+                tested.append(bases)
+                return ()
+        c = C()
+        self.assertEqual(tested, [])
+        class D(A, c, B): ...
+        self.assertEqual(tested[-1], (A, c, B))
+        self.assertEqual(D.__bases__, (A, B))
+        self.assertEqual(D.__orig_bases__, (A, c, B))
+        self.assertEqual(D.__mro__, (D, A, B, object))
+        class E(c): ...
+        self.assertEqual(tested[-1], (c,))
+        if sys.version_info[0] > 2:
+            # not all of it works on Python 2
+            self.assertEqual(E.__bases__, (object,))
+        self.assertEqual(E.__orig_bases__, (c,))
+        if sys.version_info[0] > 2:
+            # not all of it works on Python 2
+            self.assertEqual(E.__mro__, (E, object))
+
+    def test_mro_entry_with_builtins(self):
+        tested = []
+        class A: ...
+        class C:
+            def __mro_entries__(self, bases):
+                tested.append(bases)
+                return (dict,)
+        c = C()
+        self.assertEqual(tested, [])
+        class D(A, c): ...
+        self.assertEqual(tested[-1], (A, c))
+        self.assertEqual(D.__bases__, (A, dict))
+        self.assertEqual(D.__orig_bases__, (A, c))
+        self.assertEqual(D.__mro__, (D, A, dict, object))
+
+    def test_mro_entry_with_builtins_2(self):
+        tested = []
+        class C:
+            def __mro_entries__(self, bases):
+                tested.append(bases)
+                return (C,)
+        c = C()
+        self.assertEqual(tested, [])
+        class D(c, dict): ...
+        self.assertEqual(tested[-1], (c, dict))
+        self.assertEqual(D.__bases__, (C, dict))
+        self.assertEqual(D.__orig_bases__, (c, dict))
+        self.assertEqual(D.__mro__, (D, C, dict, object))
+
+    def test_mro_entry_errors(self):
+        class C_too_many:
+            def __mro_entries__(self, bases, something, other):
+                return ()
+        c = C_too_many()
+        with self.assertRaises(TypeError):
+            class D(c): ...
+        class C_too_few:
+            def __mro_entries__(self):
+                return ()
+        d = C_too_few()
+        with self.assertRaises(TypeError):
+            class D(d): ...
+
+    def test_mro_entry_errors_2(self):
+        class C_not_callable:
+            __mro_entries__ = "Surprise!"
+        c = C_not_callable()
+        with self.assertRaises(TypeError):
+            class D(c): ...
+        class C_not_tuple:
+            def __mro_entries__(self):
+                return object
+        c = C_not_tuple()
+        with self.assertRaises(TypeError):
+            class D(c): ...
+
+    def test_mro_entry_metaclass(self):
+        meta_args = []
+        class Meta(type):
+            def __new__(mcls, name, bases, ns):
+                meta_args.extend([mcls, name, bases, ns])
+                return super().__new__(mcls, name, bases, ns)
+        class A: ...
+        class C:
+            def __mro_entries__(self, bases):
+                return (A,)
+        c = C()
+        class D(c, metaclass=Meta):
+            x = 1
+        self.assertEqual(meta_args[0], Meta)
+        self.assertEqual(meta_args[1], 'D')
+        self.assertEqual(meta_args[2], (A,))
+        self.assertEqual(meta_args[3]['x'], 1)
+        self.assertEqual(D.__bases__, (A,))
+        self.assertEqual(D.__orig_bases__, (c,))
+        self.assertEqual(D.__mro__, (D, A, object))
+        self.assertEqual(D.__class__, Meta)
+
+    @unittest.skipIf(sys.version_info < (3, 7), "'type' checks for __mro_entries__ not implemented")
+    def test_mro_entry_type_call(self):
+        # Substitution should _not_ happen in direct type call
+        class C:
+            def __mro_entries__(self, bases):
+                return ()
+        c = C()
+        with self.assertRaisesRegex(TypeError,
+                                    "MRO entry resolution; "
+                                    "use types.new_class()"):
+            type('Bad', (c,), {})
+
+
+class TestClassGetitem(unittest.TestCase):
+    # BEGIN - Additional tests from cython
+    def test_no_class_getitem(self):
+        class C: ...
+        # PyPy<7.3.8 raises AttributeError on __class_getitem__
+        if hasattr(sys, "pypy_version_info")  and sys.pypy_version_info < (7, 3, 8):
+            err = AttributeError
+        else:
+            err = TypeError
+        with self.assertRaises(err):
+            C[int]
+
+    # END - Additional tests from cython
+
+    def test_class_getitem(self):
+        getitem_args = []
+        class C:
+            def __class_getitem__(*args, **kwargs):
+                getitem_args.extend([args, kwargs])
+                return None
+        C[int, str]
+        self.assertEqual(getitem_args[0], (C, (int, str)))
+        self.assertEqual(getitem_args[1], {})
+
+    def test_class_getitem_format(self):
+        class C:
+            def __class_getitem__(cls, item):
+                return f'C[{item.__name__}]'
+        self.assertEqual(C[int], 'C[int]')
+        self.assertEqual(C[C], 'C[C]')
+
+    def test_class_getitem_inheritance(self):
+        class C:
+            def __class_getitem__(cls, item):
+                return f'{cls.__name__}[{item.__name__}]'
+        class D(C): ...
+        self.assertEqual(D[int], 'D[int]')
+        self.assertEqual(D[D], 'D[D]')
+
+    def test_class_getitem_inheritance_2(self):
+        class C:
+            def __class_getitem__(cls, item):
+                return 'Should not see this'
+        class D(C):
+            def __class_getitem__(cls, item):
+                return f'{cls.__name__}[{item.__name__}]'
+        self.assertEqual(D[int], 'D[int]')
+        self.assertEqual(D[D], 'D[D]')
+
+    def test_class_getitem_classmethod(self):
+        class C:
+            @classmethod
+            def __class_getitem__(cls, item):
+                return f'{cls.__name__}[{item.__name__}]'
+        class D(C): ...
+        self.assertEqual(D[int], 'D[int]')
+        self.assertEqual(D[D], 'D[D]')
+
+    @unittest.skipIf(sys.version_info < (3, 6), "__init_subclass__() requires Py3.6+ (PEP 487)")
+    def test_class_getitem_patched(self):
+        class C:
+            def __init_subclass__(cls):
+                def __class_getitem__(cls, item):
+                    return f'{cls.__name__}[{item.__name__}]'
+                cls.__class_getitem__ = classmethod(__class_getitem__)
+        class D(C): ...
+        self.assertEqual(D[int], 'D[int]')
+        self.assertEqual(D[D], 'D[D]')
+
+    def test_class_getitem_with_builtins(self):
+        class A(dict):
+            called_with = None
+
+            def __class_getitem__(cls, item):
+                cls.called_with = item
+        class B(A):
+            pass
+        self.assertIs(B.called_with, None)
+        B[int]
+        self.assertIs(B.called_with, int)
+
+    def test_class_getitem_errors(self):
+        class C_too_few:
+            def __class_getitem__(cls):
+                return None
+        with self.assertRaises(TypeError):
+            C_too_few[int]
+        class C_too_many:
+            def __class_getitem__(cls, one, two):
+                return None
+        with self.assertRaises(TypeError):
+            C_too_many[int]
+
+    def test_class_getitem_errors_2(self):
+        class C:
+            def __class_getitem__(cls, item):
+                return None
+        with self.assertRaises(TypeError):
+            C()[int]
+        class E: ...
+        e = E()
+        e.__class_getitem__ = lambda cls, item: 'This will not work'
+        with self.assertRaises(TypeError):
+            e[int]
+        class C_not_callable:
+            __class_getitem__ = "Surprise!"
+        with self.assertRaises(TypeError):
+            C_not_callable[int]
+
+    def test_class_getitem_metaclass(self):
+        class Meta(type):
+            def __class_getitem__(cls, item):
+                return f'{cls.__name__}[{item.__name__}]'
+        self.assertEqual(Meta[int], 'Meta[int]')
+
+    def test_class_getitem_with_metaclass(self):
+        class Meta(type): pass
+        class C(metaclass=Meta):
+            def __class_getitem__(cls, item):
+                return f'{cls.__name__}[{item.__name__}]'
+        self.assertEqual(C[int], 'C[int]')
+
+    def test_class_getitem_metaclass_first(self):
+        class Meta(type):
+            def __getitem__(cls, item):
+                return 'from metaclass'
+        class C(metaclass=Meta):
+            def __class_getitem__(cls, item):
+                return 'from __class_getitem__'
+        self.assertEqual(C[int], 'from metaclass')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_grammar.py cython-0.20.1+1~202203241016-9537/tests/run/test_grammar.py
--- cython-0.20.1+1~201611251650-6686/tests/run/test_grammar.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_grammar.py	2022-03-24 10:16:46.000000000 +0000
@@ -1,9 +1,26 @@
-### COPIED FROM CPython 3.5 - ADDED PART FOLLOWS ###
+### COPIED FROM CPython 3.9 - ADDED PART FOLLOWS ###
 # cython: language_level=3
 
+import cython
 import contextlib
 from tempfile import NamedTemporaryFile
-from Cython.Compiler.Main import compile as cython_compile
+from Cython.Compiler.Main import compile as cython_compile, CompileError
+from Cython.Build.Inline import cython_inline
+
+
+@contextlib.contextmanager
+def hidden_stderr():
+    try:
+        from StringIO import StringIO
+    except ImportError:
+        from io import StringIO
+
+    old_stderr = sys.stderr
+    try:
+        sys.stderr = StringIO()
+        yield
+    finally:
+        sys.stderr = old_stderr
 
 
 def _compile(code):
@@ -11,36 +28,44 @@
         f.write(code.encode('utf8'))
         f.flush()
 
-        try:
-            from StringIO import StringIO
-        except ImportError:
-            from io import StringIO
-
-        old_stderr = sys.stderr
-        try:
-            sys.stderr = StringIO()
+        with hidden_stderr():
             result = cython_compile(f.name, language_level=3)
-        finally:
-            sys.stderr = old_stderr
     return result
 
 
-def check_syntax_error(test, code):
+def check_syntax_error(test, code, msg=None):
     result = _compile(code)
     assert not result.c_file
 
 
-def compile(code, name, what):
-    assert what == 'exec'
-    result = _compile(code)
-    if not result.c_file:
-        raise SyntaxError('unexpected EOF')  # see usage of compile() below
+def check_syntax_warning(test, code, *args):
+    # ignore for now
+    return _compile(code)
+
+
+if cython.compiled:
+    def compile(code, name, what):
+        assert what == 'exec'
+        result = _compile(code)
+        if not result.c_file:
+            raise SyntaxError('unexpected EOF')  # see usage of compile() below
+
+    def exec(code):
+        result = _compile(code)
+        if not result.c_file:
+            raise SyntaxError('unexpected EOF')  # see usage of compile() below
+
+    def eval(code):
+        try:
+            with hidden_stderr():
+                return cython_inline(code)
+        except CompileError as exc:
+            raise SyntaxError(str(exc))
 
 
-def exec(code):
-    result = _compile(code)
-    if not result.c_file:
-        raise SyntaxError('unexpected EOF')  # see usage of compile() below
+def use_old_parser():
+    # FIXME: currently disabling new PEG parser tests.
+    return True
 
 
 import unittest
@@ -58,22 +83,123 @@
 
     unittest.skipUnless = skipUnless
 
+if not hasattr(unittest, 'skip'):
+    def skip(message):
+        return unittest.skipUnless(False, message)
+
+    unittest.skip = skip
+
+skip = unittest.skip
+
 
 ### END OF CYTHON ADDED PART - COPIED PART FOLLOWS ###
 
 # Python test set -- part 1, grammar.
 # This just tests whether the parser accepts them all.
 
-#from test.support import check_syntax_error
+#from test.support import check_syntax_error, check_syntax_warning, use_old_parser
 import inspect
 import unittest
 import sys
+import warnings
 # testing import *
 from sys import *
 
+# different import patterns to check that __annotations__ does not interfere
+# with import machinery
+#import test.ann_module as ann_module
+#import typing
+#from collections import ChainMap
+#from test import ann_module2
+#import test
+
+# These are shared with test_tokenize and other test modules.
+#
+# Note: since several test cases filter out floats by looking for "e" and ".",
+# don't add hexadecimal literals that contain "e" or "E".
+VALID_UNDERSCORE_LITERALS = [
+    '0_0_0',
+    '4_2',
+    '1_0000_0000',
+    '0b1001_0100',
+    '0xffff_ffff',
+    '0o5_7_7',
+    '1_00_00.5',
+    '1_00_00.5e5',
+    '1_00_00e5_1',
+    '1e1_0',
+    '.1_4',
+    '.1_4e1',
+    '0b_0',
+    '0x_f',
+    '0o_5',
+    '1_00_00j',
+    '1_00_00.5j',
+    '1_00_00e5_1j',
+    '.1_4j',
+    '(1_2.5+3_3j)',
+    '(.5_6j)',
+]
+INVALID_UNDERSCORE_LITERALS = [
+    # Trailing underscores:
+    '0_',
+    '42_',
+    '1.4j_',
+    '0x_',
+    '0b1_',
+    '0xf_',
+    '0o5_',
+    '0 if 1_Else 1',
+    # Underscores in the base selector:
+    '0_b0',
+    '0_xf',
+    '0_o5',
+    # Old-style octal, still disallowed:
+    '0_7',
+    '09_99',
+    # Multiple consecutive underscores:
+    '4_______2',
+    '0.1__4',
+    '0.1__4j',
+    '0b1001__0100',
+    '0xffff__ffff',
+    '0x___',
+    '0o5__77',
+    '1e1__0',
+    '1e1__0j',
+    # Underscore right before a dot:
+    '1_.4',
+    '1_.4j',
+    # Underscore right after a dot:
+    '1._4',
+    '1._4j',
+    '._5',
+    '._5j',
+    # Underscore right after a sign:
+    '1.0e+_1',
+    '1.0e+_1j',
+    # Underscore right before j:
+    '1.4_j',
+    '1.4e5_j',
+    # Underscore right before e:
+    '1_e1',
+    '1.4_e1',
+    '1.4_e1j',
+    # Underscore right after e:
+    '1e_1',
+    '1.4e_1',
+    '1.4e_1j',
+    # Complex cases with parens:
+    '(1+1.5_j_)',
+    '(1+1.5_j)',
+]
+
 
 class TokenTests(unittest.TestCase):
 
+    #from test.support import check_syntax_error
+    check_syntax_error = check_syntax_error
+
     def test_backslash(self):
         # Backslash means line continuation:
         x = 1 \
@@ -150,6 +276,40 @@
         self.assertEqual(1 if 0else 0, 0)
         self.assertRaises(SyntaxError, eval, "0 if 1Else 0")
 
+    @skip("Done more efficiently in TestGrammar")
+    def test_underscore_literals(self):
+        for lit in VALID_UNDERSCORE_LITERALS:
+            self.assertEqual(eval(lit), eval(lit.replace('_', '')))
+        for lit in INVALID_UNDERSCORE_LITERALS:
+            self.assertRaises(SyntaxError, eval, lit)
+        # Sanity check: no literal begins with an underscore
+        self.assertRaises(NameError, eval, "_0")
+
+    def test_bad_numerical_literals(self):
+        check = self.check_syntax_error
+        check("0b12", "invalid digit '2' in binary literal")
+        check("0b1_2", "invalid digit '2' in binary literal")
+        check("0b2", "invalid digit '2' in binary literal")
+        check("0b1_", "invalid binary literal")
+        check("0b", "invalid binary literal")
+        check("0o18", "invalid digit '8' in octal literal")
+        check("0o1_8", "invalid digit '8' in octal literal")
+        check("0o8", "invalid digit '8' in octal literal")
+        check("0o1_", "invalid octal literal")
+        check("0o", "invalid octal literal")
+        check("0x1_", "invalid hexadecimal literal")
+        check("0x", "invalid hexadecimal literal")
+        check("1_", "invalid decimal literal")
+        # FIXME: not currently a syntax error
+        """
+        check("012",
+              "leading zeros in decimal integer literals are not permitted; "
+              "use an 0o prefix for octal integers")
+        """
+        check("1.2_", "invalid decimal literal")
+        check("1e2_", "invalid decimal literal")
+        check("1e+", "invalid decimal literal")
+
     def test_string_literals(self):
         x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y)
         x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39)
@@ -193,7 +353,8 @@
     def test_ellipsis(self):
         x = ...
         self.assertTrue(x is Ellipsis)
-        self.assertRaises(SyntaxError, eval, ".. .")
+        # FIXME: why is this not rejected ???
+        #self.assertRaises(SyntaxError, eval, ".. .")
 
     def test_eof_error(self):
         samples = ("def foo(", "\ndef foo(", "def foo(\n")
@@ -202,8 +363,33 @@
                 compile(s, "", "exec")
             self.assertIn("unexpected EOF", str(cm.exception))
 
+var_annot_global: int  # a global annotated is necessary for test_var_annot
+
+# custom namespace for testing __annotations__
+
+class CNS:
+    def __init__(self):
+        self._dct = {}
+    def __setitem__(self, item, value):
+        self._dct[item.lower()] = value
+    def __getitem__(self, item):
+        return self._dct[item]
+
+
 class GrammarTests(unittest.TestCase):
 
+    #from test.support import check_syntax_error, check_syntax_warning
+    check_syntax_error, check_syntax_warning = check_syntax_error, check_syntax_warning
+
+    if not hasattr(unittest.TestCase, 'subTest'):
+        @contextlib.contextmanager
+        def subTest(self, source=None, case=None, **kwargs):
+            try:
+                yield
+            except Exception:
+                print(source or case)
+                raise
+
     # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
     # XXX can't test in a script -- this rule is only used when interactive
 
@@ -217,9 +403,188 @@
         # testlist ENDMARKER
         x = eval('1, 0 or 1')
 
+    def test_var_annot_basics(self):
+        # all these should be allowed
+        var1: int = 5
+        var2: [int, str]
+        my_lst = [42]
+        def one():
+            return 1
+        int.new_attr: int
+        [list][0]: type
+        my_lst[one()-1]: int = 5
+        self.assertEqual(my_lst, [5])
+
+    @skip("Cython Bug: global vs. local declarations do not currently raise an error")
+    def test_var_annot_syntax_errors(self):
+        # parser pass
+        check_syntax_error(self, "def f: int")
+        check_syntax_error(self, "x: int: str")
+        check_syntax_error(self, "def f():\n"
+                                 "    nonlocal x: int\n")
+        # AST pass
+        check_syntax_error(self, "[x, 0]: int\n")
+        check_syntax_error(self, "f(): int\n")
+        check_syntax_error(self, "(x,): int")
+        check_syntax_error(self, "def f():\n"
+                                 "    (x, y): int = (1, 2)\n")
+        # symtable pass
+        check_syntax_error(self, "def f():\n"
+                                 "    x: int\n"
+                                 "    global x\n")
+        check_syntax_error(self, "def f():\n"
+                                 "    global x\n"
+                                 "    x: int\n")
+
+    def test_var_annot_basic_semantics(self):
+        # execution order
+        with self.assertRaises(ZeroDivisionError):
+            no_name[does_not_exist]: no_name_again = 1/0
+        with self.assertRaises(NameError):
+            no_name[does_not_exist]: 1/0 = 0
+        global var_annot_global
+
+        # function semantics
+        def f():
+            st: str = "Hello"
+            a.b: int = (1, 2)
+            return st
+        self.assertEqual(f.__annotations__, {})
+        def f_OK():
+            x: 1/0
+        f_OK()
+        # Compile-time errors in Cython:
+        """
+        def fbad():
+            x: int
+            print(x)
+        with self.assertRaises(UnboundLocalError):
+            fbad()
+        def f2bad():
+            (no_such_global): int
+            print(no_such_global)
+        try:
+            f2bad()
+        except Exception as e:
+            self.assertIs(type(e), NameError)
+        """
+
+        # class semantics
+        class C:
+            __foo: int
+            s: str = "attr"
+            z = 2
+            def __init__(self, x):
+                self.x: int = x
+
+        self.assertEqual(C.__annotations__, {'_C__foo': 'int', 's': 'str'})
+        with self.assertRaises(NameError):
+            class CBad:
+                no_such_name_defined.attr: int = 0
+        with self.assertRaises(NameError):
+            class Cbad2(C):
+                x: int
+                x.y: list = []
+
+    @skip("Not currently supported: https://github.com/cython/cython/issues/3839")
+    def test_var_annot_metaclass_semantics(self):
+        class CMeta(type):
+            @classmethod
+            def __prepare__(metacls, name, bases, **kwds):
+                return {'__annotations__': CNS()}
+        class CC(metaclass=CMeta):
+            XX: 'ANNOT'
+        self.assertEqual(CC.__annotations__['xx'], 'ANNOT')
+
+    @skip("Depends on external test module")
+    def test_var_annot_module_semantics(self):
+        with self.assertRaises(AttributeError):
+            print(test.__annotations__)
+        self.assertEqual(ann_module.__annotations__,
+                     {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int]})
+        self.assertEqual(ann_module.M.__annotations__,
+                              {'123': 123, 'o': type})
+        self.assertEqual(ann_module2.__annotations__, {})
+
+    @skip("Depends on external test module")
+    def test_var_annot_in_module(self):
+        # check that functions fail the same way when executed
+        # outside of module where they were defined
+        from test.ann_module3 import f_bad_ann, g_bad_ann, D_bad_ann
+        with self.assertRaises(NameError):
+            f_bad_ann()
+        with self.assertRaises(NameError):
+            g_bad_ann()
+        with self.assertRaises(NameError):
+            D_bad_ann(5)
+
+    @skip("Depends on 3-args compiled exec()")
+    def test_var_annot_simple_exec(self):
+        gns = {}; lns= {}
+        exec("'docstring'\n"
+             "__annotations__[1] = 2\n"
+             "x: int = 5\n", gns, lns)
+        self.assertEqual(lns["__annotations__"], {1: 2, 'x': int})
+        with self.assertRaises(KeyError):
+            gns['__annotations__']
+
+    @skip("Depends on 3-args compiled exec()")
+    def test_var_annot_custom_maps(self):
+        # tests with custom locals() and __annotations__
+        ns = {'__annotations__': CNS()}
+        exec('X: int; Z: str = "Z"; (w): complex = 1j', ns)
+        self.assertEqual(ns['__annotations__']['x'], int)
+        self.assertEqual(ns['__annotations__']['z'], str)
+        with self.assertRaises(KeyError):
+            ns['__annotations__']['w']
+        nonloc_ns = {}
+        class CNS2:
+            def __init__(self):
+                self._dct = {}
+            def __setitem__(self, item, value):
+                nonlocal nonloc_ns
+                self._dct[item] = value
+                nonloc_ns[item] = value
+            def __getitem__(self, item):
+                return self._dct[item]
+        exec('x: int = 1', {}, CNS2())
+        self.assertEqual(nonloc_ns['__annotations__']['x'], int)
+
+    @skip("Depends on 3-args compiled exec()")
+    def test_var_annot_refleak(self):
+        # complex case: custom locals plus custom __annotations__
+        # this was causing refleak
+        cns = CNS()
+        nonloc_ns = {'__annotations__': cns}
+        class CNS2:
+            def __init__(self):
+                self._dct = {'__annotations__': cns}
+            def __setitem__(self, item, value):
+                nonlocal nonloc_ns
+                self._dct[item] = value
+                nonloc_ns[item] = value
+            def __getitem__(self, item):
+                return self._dct[item]
+        exec('X: str', {}, CNS2())
+        self.assertEqual(nonloc_ns['__annotations__']['x'], str)
+
+    @skip("Depends on 3-args compiled exec()")
+    def test_var_annot_rhs(self):
+        ns = {}
+        exec('x: tuple = 1, 2', ns)
+        self.assertEqual(ns['x'], (1, 2))
+        stmt = ('def f():\n'
+                '    x: int = yield')
+        exec(stmt, ns)
+        self.assertEqual(list(ns['f']()), [None])
+
+        ns = {"a": 1, 'b': (2, 3, 4), "c":5, "Tuple": typing.Tuple}
+        exec('x: Tuple[int, ...] = a,*b,c', ns)
+        self.assertEqual(ns['x'], (1, 2, 3, 4, 5))
+
     def test_funcdef(self):
         ### [decorators] 'def' NAME parameters ['->' test] ':' suite
-        ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        ### decorator: '@' namedexpr_test NEWLINE
         ### decorators: decorator+
         ### parameters: '(' [typedargslist] ')'
         ### typedargslist: ((tfpdef ['=' test] ',')*
@@ -358,9 +723,10 @@
         pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200)
         pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100)
 
-        self.assertRaises(SyntaxError, eval, "def f(*): pass")
-        self.assertRaises(SyntaxError, eval, "def f(*,): pass")
-        self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass")
+        # FIXME: currently does not raise an error
+        #self.assertRaises(SyntaxError, eval, "def f(*): pass")
+        #self.assertRaises(SyntaxError, eval, "def f(*,): pass")
+        #self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass")
 
         # keyword arguments after *arglist
         def f(*args, **kwargs):
@@ -374,51 +740,87 @@
         self.assertEqual(f(spam='fried', **{'eggs':'scrambled'}),
                          ((), {'eggs':'scrambled', 'spam':'fried'}))
 
+        # Check ast errors in *args and *kwargs
+        check_syntax_error(self, "f(*g(1=2))")
+        check_syntax_error(self, "f(**g(1=2))")
+
         # argument annotation tests
         def f(x) -> list: pass
-        self.assertEqual(f.__annotations__, {'return': list})
+        self.assertEqual(f.__annotations__, {'return': 'list'})
         def f(x: int): pass
-        self.assertEqual(f.__annotations__, {'x': int})
+        self.assertEqual(f.__annotations__, {'x': 'int'})
+        def f(x: int, /): pass
+        self.assertEqual(f.__annotations__, {'x': 'int'})
+        def f(x: int = 34, /): pass
+        self.assertEqual(f.__annotations__, {'x': 'int'})
         def f(*x: str): pass
-        self.assertEqual(f.__annotations__, {'x': str})
+        self.assertEqual(f.__annotations__, {'x': 'str'})
         def f(**x: float): pass
-        self.assertEqual(f.__annotations__, {'x': float})
+        self.assertEqual(f.__annotations__, {'x': 'float'})
         def f(x, y: 1+2): pass
-        self.assertEqual(f.__annotations__, {'y': 3})
+        self.assertEqual(f.__annotations__, {'y': '1 + 2'})
+        def f(x, y: 1+2, /): pass
+        self.assertEqual(f.__annotations__, {'y': '1 + 2'})
         def f(a, b: 1, c: 2, d): pass
-        self.assertEqual(f.__annotations__, {'b': 1, 'c': 2})
+        self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'})
+        def f(a, b: 1, /, c: 2, d): pass
+        self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'})
         def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6): pass
         self.assertEqual(f.__annotations__,
-                         {'b': 1, 'c': 2, 'e': 3, 'g': 6})
+                         {'b': '1', 'c': '2', 'e': '3', 'g': '6'})
         def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6, h: 7, i=8, j: 9 = 10,
               **k: 11) -> 12: pass
         self.assertEqual(f.__annotations__,
-                         {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
-                          'k': 11, 'return': 12})
+                         {'b': '1', 'c': '2', 'e': '3', 'g': '6', 'h': '7', 'j': '9',
+                          'k': '11', 'return': '12'})
+        # FIXME: compile failure on positional-only argument declaration
+        """
+        def f(a, b: 1, c: 2, d, e: 3 = 4, f: int = 5, /, *g: 6, h: 7, i=8, j: 9 = 10,
+              **k: 11) -> 12: pass
+        self.assertEqual(f.__annotations__,
+                          {'b': 1, 'c': 2, 'e': 3, 'f': int, 'g': 6, 'h': 7, 'j': 9,
+                           'k': 11, 'return': 12})
+        """
         # Check for issue #20625 -- annotations mangling
         class Spam:
             def f(self, *, __kw: 1):
                 pass
         class Ham(Spam): pass
-        self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': 1})
-        self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': 1})
+        # FIXME: not currently mangled
+        """
+        self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': '1'})
+        self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': '1'})
+        """
         # Check for SF Bug #1697248 - mixing decorators and a return annotation
         def null(x): return x
         @null
         def f(x) -> list: pass
-        self.assertEqual(f.__annotations__, {'return': list})
+        self.assertEqual(f.__annotations__, {'return': 'list'})
+
+        # Test expressions as decorators (PEP 614):
+        # FIXME: implement PEP 614
+        """
+        @False or null
+        def f(x): pass
+        @d := null
+        def f(x): pass
+        @lambda f: null(f)
+        def f(x): pass
+        @[..., null, ...][1]
+        def f(x): pass
+        @null(null)(null)
+        def f(x): pass
+        @[null][0].__call__.__call__
+        def f(x): pass
+        """
 
-        # test MAKE_CLOSURE with a variety of oparg's
+        # test closures with a variety of opargs
         closure = 1
         def f(): return closure
         def f(x=1): return closure
         def f(*, k=1): return closure
         def f() -> int: return closure
 
-        # Check ast errors in *args and *kwargs
-        check_syntax_error(self, "f(*g(1=2))")
-        check_syntax_error(self, "f(**g(1=2))")
-
         # Check trailing commas are permitted in funcdef argument list
         def f(a,): pass
         def f(*args,): pass
@@ -440,7 +842,7 @@
         ### lambdef: 'lambda' [varargslist] ':' test
         l1 = lambda : 0
         self.assertEqual(l1(), 0)
-        l2 = lambda : a[d] # XXX just testing the expression
+        l2 = lambda : a[d]  # XXX just testing the expression
         l3 = lambda : [2 < x for x in [-1, 3, 0]]
         self.assertEqual(l3(), [0, 1, 0])
         l4 = lambda x = lambda y = lambda z=1 : z : y() : x()
@@ -517,11 +919,13 @@
             for case in cases:
                 source = case.format(keyword)
                 with self.subTest(source=source):
-                    with self.assertRaisesRegex(SyntaxError, custom_msg):
+                    #with self.assertRaisesRegex(SyntaxError, custom_msg):
+                    with self.assertRaises(SyntaxError):
                         exec(source)
                 source = source.replace("foo", "(foo.)")
                 with self.subTest(source=source):
-                    with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+                    #with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+                    with self.assertRaises(SyntaxError):
                         exec(source)
 
     def test_del_stmt(self):
@@ -533,6 +937,24 @@
         del abc
         del x, y, (z, xyz)
 
+        x, y, z = "xyz"
+        del x
+        del y,
+        del (z)
+        del ()
+
+        a, b, c, d, e, f, g = "abcdefg"
+        del a, (b, c), (d, (e, f))
+
+        a, b, c, d, e, f, g = "abcdefg"
+        del a, [b, c], (d, [e, f])
+
+        abcd = list("abcd")
+        del abcd[1:2]
+
+        # FIXME: currently fails to compile
+        #compile("del a, (b[0].c, (d.e, f.g[1:2])), [h.i.j], ()", "", "exec")
+
     def test_pass_stmt(self):
         # 'pass'
         pass
@@ -597,13 +1019,202 @@
         test_inner()
 
     def test_return(self):
-        # 'return' [testlist]
+        # 'return' [testlist_star_expr]
         def g1(): return
         def g2(): return 1
+        def g3():
+            z = [2, 3]
+            return 1, *z
+
         g1()
         x = g2()
+        y = g3()
+        self.assertEqual(y, (1, 2, 3), "unparenthesized star expr return")
         check_syntax_error(self, "class foo:return 1")
 
+    def test_break_in_finally(self):
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                pass
+            finally:
+                break
+        self.assertEqual(count, 1)
+
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                continue
+            finally:
+                break
+        self.assertEqual(count, 1)
+
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                1/0
+            finally:
+                break
+        self.assertEqual(count, 1)
+
+        for count in [0, 1]:
+            self.assertEqual(count, 0)
+            try:
+                pass
+            finally:
+                break
+        self.assertEqual(count, 0)
+
+        for count in [0, 1]:
+            self.assertEqual(count, 0)
+            try:
+                continue
+            finally:
+                break
+        self.assertEqual(count, 0)
+
+        for count in [0, 1]:
+            self.assertEqual(count, 0)
+            try:
+                1/0
+            finally:
+                break
+        self.assertEqual(count, 0)
+
+    def test_continue_in_finally(self):
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                pass
+            finally:
+                continue
+            break
+        self.assertEqual(count, 2)
+
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                break
+            finally:
+                continue
+        self.assertEqual(count, 2)
+
+        count = 0
+        while count < 2:
+            count += 1
+            try:
+                1/0
+            finally:
+                continue
+            break
+        self.assertEqual(count, 2)
+
+        for count in [0, 1]:
+            try:
+                pass
+            finally:
+                continue
+            break
+        self.assertEqual(count, 1)
+
+        for count in [0, 1]:
+            try:
+                break
+            finally:
+                continue
+        self.assertEqual(count, 1)
+
+        for count in [0, 1]:
+            try:
+                1/0
+            finally:
+                continue
+            break
+        self.assertEqual(count, 1)
+
+    def test_return_in_finally(self):
+        def g1():
+            try:
+                pass
+            finally:
+                return 1
+        self.assertEqual(g1(), 1)
+
+        def g2():
+            try:
+                return 2
+            finally:
+                return 3
+        self.assertEqual(g2(), 3)
+
+        def g3():
+            try:
+                1/0
+            finally:
+                return 4
+        self.assertEqual(g3(), 4)
+
+    @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit")
+    def test_break_in_finally_after_return(self):
+        # See issue #37830
+        def g1(x):
+            for count in [0, 1]:
+                count2 = 0
+                while count2 < 20:
+                    count2 += 10
+                    try:
+                        return count + count2
+                    finally:
+                        if x:
+                            break
+            return 'end', count, count2
+        self.assertEqual(g1(False), 10)
+        self.assertEqual(g1(True), ('end', 1, 10))
+
+        def g2(x):
+            for count in [0, 1]:
+                for count2 in [10, 20]:
+                    try:
+                        return count + count2
+                    finally:
+                        if x:
+                            break
+            return 'end', count, count2
+        self.assertEqual(g2(False), 10)
+        self.assertEqual(g2(True), ('end', 1, 10))
+
+    @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit")
+    def test_continue_in_finally_after_return(self):
+        # See issue #37830
+        def g1(x):
+            count = 0
+            while count < 100:
+                count += 1
+                try:
+                    return count
+                finally:
+                    if x:
+                        continue
+            return 'end', count
+        self.assertEqual(g1(False), 1)
+        self.assertEqual(g1(True), ('end', 100))
+
+        def g2(x):
+            for count in [0, 1]:
+                try:
+                    return count
+                finally:
+                    if x:
+                        continue
+            return 'end', count
+        self.assertEqual(g2(False), 0)
+        self.assertEqual(g2(True), ('end', 1))
+
     def test_yield(self):
         # Allowed as standalone statement
         def g(): yield 1
@@ -627,6 +1238,9 @@
         def g(): f((yield 1), 1)
         def g(): f((yield from ()))
         def g(): f((yield from ()), 1)
+        # Do not require parenthesis for tuple unpacking
+        def g(): rest = 4, 5, 6; yield 1, 2, 3, *rest
+        self.assertEqual(list(g()), [(1, 2, 3, 4, 5, 6)])
         check_syntax_error(self, "def g(): f(yield 1)")
         check_syntax_error(self, "def g(): f(yield 1, 1)")
         check_syntax_error(self, "def g(): f(yield from ())")
@@ -638,7 +1252,35 @@
         check_syntax_error(self, "class foo:yield 1")
         check_syntax_error(self, "class foo:yield from ()")
         # Check annotation refleak on SyntaxError
-        check_syntax_error(self, "def g(a:(yield)): pass")
+        #check_syntax_error(self, "def g(a:(yield)): pass")  # no longer a syntax error with PEP-563
+
+    @skip("Not currently a syntax error")
+    def test_yield_in_comprehensions(self):
+        # Check yield in comprehensions
+        def g(): [x for x in [(yield 1)]]
+        def g(): [x for x in [(yield from ())]]
+
+        check = self.check_syntax_error
+        check("def g(): [(yield x) for x in ()]",
+              "'yield' inside list comprehension")
+        check("def g(): [x for x in () if not (yield x)]",
+              "'yield' inside list comprehension")
+        check("def g(): [y for x in () for y in [(yield x)]]",
+              "'yield' inside list comprehension")
+        check("def g(): {(yield x) for x in ()}",
+              "'yield' inside set comprehension")
+        check("def g(): {(yield x): x for x in ()}",
+              "'yield' inside dict comprehension")
+        check("def g(): {x: (yield x) for x in ()}",
+              "'yield' inside dict comprehension")
+        check("def g(): ((yield x) for x in ())",
+              "'yield' inside generator expression")
+        check("def g(): [(yield from x) for x in ()]",
+              "'yield' inside list comprehension")
+        check("class C: [(yield x) for x in ()]",
+              "'yield' inside list comprehension")
+        check("[(yield x) for x in ()]",
+              "'yield' inside list comprehension")
 
     def test_raise(self):
         # 'raise' test [',' test]
@@ -709,6 +1351,15 @@
         else:
             self.fail("AssertionError not raised by 'assert False'")
 
+        self.check_syntax_warning('assert(x, "msg")',
+                                  'assertion is always true')
+        # FIXME: currently fails to compile
+        """
+        with warnings.catch_warnings():
+            warnings.simplefilter('error', SyntaxWarning)
+            compile('assert x, "msg"', '', 'exec')
+        """
+
 
     ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef
     # Tested below
@@ -771,7 +1422,7 @@
     def test_try(self):
         ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
         ###         | 'try' ':' suite 'finally' ':' suite
-        ### except_clause: 'except' [expr ['as' expr]]
+        ### except_clause: 'except' [expr ['as' NAME]]
         try:
             1/0
         except ZeroDivisionError:
@@ -781,7 +1432,6 @@
         try: 1/0
         except EOFError: pass
         except TypeError as msg: pass
-        except RuntimeError as msg: pass
         except: pass
         else: pass
         try: 1/0
@@ -790,6 +1440,9 @@
         except (EOFError, TypeError, ZeroDivisionError) as msg: pass
         try: pass
         finally: pass
+        with self.assertRaises(SyntaxError):
+            compile("try:\n    pass\nexcept Exception as a.b:\n    pass", "?", "exec")
+            compile("try:\n    pass\nexcept Exception as a[b]:\n    pass", "?", "exec")
 
     def test_suite(self):
         # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT
@@ -828,11 +1481,116 @@
         if 1 > 1: pass
         if 1 <= 1: pass
         if 1 >= 1: pass
-        if 1 is 1: pass
-        if 1 is not 1: pass
+        if x is x: pass
+        if x is not x: pass
         if 1 in (): pass
         if 1 not in (): pass
-        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass
+        if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in x is x is not x: pass
+
+    def test_comparison_is_literal(self):
+        def check(test, msg='"is" with a literal'):
+            self.check_syntax_warning(test, msg)
+
+        check('x is 1')
+        check('x is "thing"')
+        check('1 is x')
+        check('x is y is 1')
+        check('x is not 1', '"is not" with a literal')
+
+        # FIXME: this fails to compile
+        """
+        with warnings.catch_warnings():
+            warnings.simplefilter('error', SyntaxWarning)
+            compile('x is None', '', 'exec')
+            compile('x is False', '', 'exec')
+            compile('x is True', '', 'exec')
+            compile('x is ...', '', 'exec')
+        """
+
+    def test_warn_missed_comma(self):
+        # FIXME: would be nice if this could actually raise a compile time warning as well
+        def check(test):
+            self.check_syntax_warning(test, msg)
+
+        msg=r'is not callable; perhaps you missed a comma\?'
+        check('[(1, 2) (3, 4)]')
+        check('[(x, y) (3, 4)]')
+        check('[[1, 2] (3, 4)]')
+        check('[{1, 2} (3, 4)]')
+        check('[{1: 2} (3, 4)]')
+        check('[[i for i in range(5)] (3, 4)]')
+        check('[{i for i in range(5)} (3, 4)]')
+        check('[(i for i in range(5)) (3, 4)]')
+        check('[{i: i for i in range(5)} (3, 4)]')
+        check('[f"{x}" (3, 4)]')
+        check('[f"x={x}" (3, 4)]')
+        check('["abc" (3, 4)]')
+        check('[b"abc" (3, 4)]')
+        check('[123 (3, 4)]')
+        check('[12.3 (3, 4)]')
+        check('[12.3j (3, 4)]')
+        check('[None (3, 4)]')
+        check('[True (3, 4)]')
+        check('[... (3, 4)]')
+
+        msg=r'is not subscriptable; perhaps you missed a comma\?'
+        check('[{1, 2} [i, j]]')
+        check('[{i for i in range(5)} [i, j]]')
+        check('[(i for i in range(5)) [i, j]]')
+        check('[(lambda x, y: x) [i, j]]')
+        check('[123 [i, j]]')
+        check('[12.3 [i, j]]')
+        check('[12.3j [i, j]]')
+        check('[None [i, j]]')
+        check('[True [i, j]]')
+        check('[... [i, j]]')
+
+        msg=r'indices must be integers or slices, not tuple; perhaps you missed a comma\?'
+        check('[(1, 2) [i, j]]')
+        check('[(x, y) [i, j]]')
+        check('[[1, 2] [i, j]]')
+        check('[[i for i in range(5)] [i, j]]')
+        check('[f"{x}" [i, j]]')
+        check('[f"x={x}" [i, j]]')
+        check('["abc" [i, j]]')
+        check('[b"abc" [i, j]]')
+
+        msg=r'indices must be integers or slices, not tuple;'
+        check('[[1, 2] [3, 4]]')
+        msg=r'indices must be integers or slices, not list;'
+        check('[[1, 2] [[3, 4]]]')
+        check('[[1, 2] [[i for i in range(5)]]]')
+        msg=r'indices must be integers or slices, not set;'
+        check('[[1, 2] [{3, 4}]]')
+        check('[[1, 2] [{i for i in range(5)}]]')
+        msg=r'indices must be integers or slices, not dict;'
+        check('[[1, 2] [{3: 4}]]')
+        check('[[1, 2] [{i: i for i in range(5)}]]')
+        msg=r'indices must be integers or slices, not generator;'
+        check('[[1, 2] [(i for i in range(5))]]')
+        msg=r'indices must be integers or slices, not function;'
+        check('[[1, 2] [(lambda x, y: x)]]')
+        msg=r'indices must be integers or slices, not str;'
+        check('[[1, 2] [f"{x}"]]')
+        check('[[1, 2] [f"x={x}"]]')
+        check('[[1, 2] ["abc"]]')
+        msg=r'indices must be integers or slices, not'
+        check('[[1, 2] [b"abc"]]')
+        check('[[1, 2] [12.3]]')
+        check('[[1, 2] [12.3j]]')
+        check('[[1, 2] [None]]')
+        check('[[1, 2] [...]]')
+
+        """
+        with warnings.catch_warnings():
+            warnings.simplefilter('error', SyntaxWarning)
+            compile('[(lambda x, y: x) (3, 4)]', '', 'exec')
+            compile('[[1, 2] [i]]', '', 'exec')
+            compile('[[1, 2] [0]]', '', 'exec')
+            compile('[[1, 2] [True]]', '', 'exec')
+            compile('[[1, 2] [1:2]]', '', 'exec')
+            compile('[{(1, 2): 3} [i, j]]', '', 'exec')
+        """
 
     def test_binary_mask_ops(self):
         x = 1 & 1
@@ -890,7 +1648,7 @@
         d[1,2] = 3
         d[1,2,3] = 4
         L = list(d)
-        L.sort(key=lambda x: x if isinstance(x, tuple) else ())
+        L.sort(key=lambda x: (type(x).__name__, x))
         self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]')
 
     def test_atoms(self):
@@ -940,13 +1698,30 @@
             def meth2(self, arg): pass
             def meth3(self, a1, a2): pass
 
-        # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+        # decorator: '@' namedexpr_test NEWLINE
         # decorators: decorator+
         # decorated: decorators (classdef | funcdef)
         def class_decorator(x): return x
         @class_decorator
         class G: pass
 
+        # Test expressions as decorators (PEP 614):
+        # FIXME: implement PEP 614
+        """
+        @False or class_decorator
+        class H: pass
+        @d := class_decorator
+        class I: pass
+        @lambda c: class_decorator(c)
+        class J: pass
+        @[..., class_decorator, ...][1]
+        class K: pass
+        @class_decorator(class_decorator)(class_decorator)
+        class L: pass
+        @[class_decorator][0].__call__.__call__
+        class M: pass
+        """
+
     def test_dictcomps(self):
         # dictorsetmaker: ( (test ':' test (comp_for |
         #                                   (',' test ':' test)* [','])) |
@@ -1043,7 +1818,7 @@
         self.assertEqual(sum(b), sum([x for x in range(10)]))
 
         self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)]))
-        self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2]))
+        self.assertEqual(sum(x*x for x in range(10) if x % 2), sum([x*x for x in range(10) if x % 2]))
         self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)]))
         self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)]))
         self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)]))
@@ -1054,6 +1829,8 @@
 
     def test_comprehension_specials(self):
         # test for outmost iterable precomputation
+        # FIXME: https://github.com/cython/cython/issues/1159
+        """
         x = 10; g = (i for i in range(x)); x = 5
         self.assertEqual(len(list(g)), 10)
 
@@ -1061,6 +1838,7 @@
         x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
         x = 5; t = True;
         self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+        """
 
         # Grammar allows multiple adjacent 'if's in listcomps and genexps,
         # even though it's silly. Make sure it works (ifelse broke this.)
@@ -1091,11 +1869,75 @@
         with manager() as x, manager():
             pass
 
+        if not use_old_parser():
+            test_cases = [
+                """if 1:
+                    with (
+                        manager()
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as x
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as (x, y),
+                        manager() as z,
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager(),
+                        manager()
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as x,
+                        manager() as y
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as x,
+                        manager()
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as x,
+                        manager() as y,
+                        manager() as z,
+                    ):
+                        pass
+                """,
+                """if 1:
+                    with (
+                        manager() as x,
+                        manager() as y,
+                        manager(),
+                    ):
+                        pass
+                """,
+            ]
+            for case in test_cases:
+                with self.subTest(case=case):
+                    compile(case, "", "exec")
+
+
     def test_if_else_expr(self):
         # Test ifelse expressions in various cases
         def _checkeval(msg, ret):
             "helper to check that evaluation of expressions is done correctly"
-            print(x)
+            print(msg)
             return ret
 
         # the next line is not allowed anymore
@@ -1122,9 +1964,11 @@
         self.assertEqual(16 // (4 // 2), 8)
         self.assertEqual((16 // 4) // 2, 2)
         self.assertEqual(16 // 4 // 2, 2)
-        self.assertTrue(False is (2 is 3))
-        self.assertFalse((False is 2) is 3)
-        self.assertFalse(False is 2 is 3)
+        x = 2
+        y = 3
+        self.assertTrue(False is (x is y))
+        self.assertFalse((False is x) is y)
+        self.assertFalse(False is x is y)
 
     def test_matrix_mul(self):
         # This is not intended to be a comprehensive test, rather just to be few
@@ -1141,18 +1985,6 @@
         self.assertEqual(m.other, 42)
 
     def test_async_await(self):
-        async = 1
-        await = 2
-        self.assertEqual(async, 1)
-
-        def async():
-            nonlocal await
-            await = 10
-        async()
-        self.assertEqual(await, 10)
-
-        #self.assertFalse(bool(async.__code__.co_flags & inspect.CO_COROUTINE))
-
         async def test():
             def sum():
                 pass
@@ -1177,7 +2009,7 @@
         class Done(Exception): pass
 
         class AIter:
-            async def __aiter__(self):
+            def __aiter__(self):
                 return self
             async def __anext__(self):
                 raise StopAsyncIteration
@@ -1224,54 +2056,5 @@
             foo().send(None)
 
 
-### END OF COPY ###
-
-GrammarTests.assertRaisesRegex = lambda self, exc, msg: self.assertRaises(exc)
-
-if sys.version_info < (2, 7):
-    def assertRaises(self, exc_type, func=None, *args, **kwargs):
-        if func is not None:
-            return unittest.TestCase.assertRaises(self, exc_type, func, *args, **kwargs)
-        @contextlib.contextmanager
-        def assertRaisesCM():
-            class Result(object):
-                exception = exc_type("unexpected EOF")  # see usage above
-            try:
-                yield Result()
-            except exc_type:
-                self.assertTrue(True)
-            else:
-                self.assertTrue(False)
-        return assertRaisesCM()
-    GrammarTests.assertRaises = assertRaises
-    TokenTests.assertRaises = assertRaises
-
-
-if not hasattr(unittest.TestCase, 'subTest'):
-    @contextlib.contextmanager
-    def subTest(self, source, **kwargs):
-        try:
-            yield
-        except Exception:
-            print(source)
-            raise
-    GrammarTests.subTest = subTest
-
-
-if not hasattr(unittest.TestCase, 'assertIn'):
-    def assertIn(self, member, container, msg=None):
-        self.assertTrue(member in container, msg)
-    TokenTests.assertIn = assertIn
-
-
-# FIXME: disabling some tests for real Cython bugs here
-del GrammarTests.test_comprehension_specials  # iterable pre-calculation in generator expression
-del GrammarTests.test_funcdef  # annotation mangling
-
-# this test is difficult to enable in Py2.6
-if sys.version_info < (2,7):
-    del GrammarTests.test_former_statements_refer_to_builtins
-
-
 if __name__ == '__main__':
     unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_named_expressions.py cython-0.20.1+1~202203241016-9537/tests/run/test_named_expressions.py
--- cython-0.20.1+1~201611251650-6686/tests/run/test_named_expressions.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_named_expressions.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,558 @@
+# mode: run
+# tag: pure38, no-cpp
+
+# copied from cpython with minimal modifications (mainly exec->cython_inline, and a few exception strings)
+# This is not currently run in C++ because all the cython_inline compilations fail for reasons that are unclear
+# FIXME pure38 seems to be ignored
+# cython: language_level=3
+
+import os
+import unittest
+import cython
+from Cython.Compiler.Main import CompileError
+from Cython.Build.Inline import cython_inline
+import sys
+
+if cython.compiled:
+    class StdErrHider:
+        def __enter__(self):
+            try:
+                from StringIO import StringIO
+            except ImportError:
+                from io import StringIO
+
+            self.old_stderr = sys.stderr
+            self.new_stderr = StringIO()
+            sys.stderr = self.new_stderr
+
+            return self
+
+        def __exit__(self, exc_type, exc_value, traceback):
+            sys.stderr = self.old_stderr
+
+        @property
+        def stderr_contents(self):
+            return self.new_stderr.getvalue()
+
+    def exec(code, globals_=None, locals_=None):
+        if locals_ and globals_ and (locals_ is not globals_):
+            # a hacky attempt to treat as a class definition
+            code = "class Cls:\n" + "\n".join(
+                "    " + line for line in code.split("\n"))
+        code += "\nreturn globals(), locals()"  # so we can inspect it for changes, overriding the default cython_inline behaviour
+        try:
+            with StdErrHider() as stderr_handler:
+                try:
+                    g, l = cython_inline(code, globals=globals_, locals=locals_)
+                finally:
+                    err_messages = stderr_handler.stderr_contents
+            if globals_ is not None:
+                # because Cython inline bundles everything into a function some values that
+                # we'd expect to be in globals end up in locals. This isn't quite right but is
+                # as close as it's possible to get to retrieving the values
+                globals_.update(l)
+                globals_.update(g)
+        except CompileError as exc:
+            raised_message = str(exc)
+            if raised_message.endswith(".pyx"):
+                # unhelpfully Cython sometimes raises a compile error and sometimes just raises the filename
+                raised_message = []
+                for line in err_messages.split("\n"):
+                    line = line.split(":",3)
+                    # a usable error message with be filename:line:char: message
+                    if len(line) == 4 and line[0].endswith(".pyx"):
+                        raised_message.append(line[-1])
+                # output all the errors - we aren't worried about reproducing the exact order CPython
+                # emits errors in
+                raised_message = "; ".join(raised_message)
+            raise SyntaxError(raised_message) from None
+
+if sys.version_info[0] < 3:
+    # some monkey patching
+    unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
+
+    class FakeSubTest(object):
+        def __init__(self, *args, **kwds):
+            pass
+        def __enter__(self):
+            pass
+        def __exit__(self, *args):
+            pass
+    unittest.TestCase.subTest = FakeSubTest
+
+class NamedExpressionInvalidTest(unittest.TestCase):
+
+    def test_named_expression_invalid_01(self):
+        code = """x := 0"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_02(self):
+        code = """x = y := 0"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_03(self):
+        code = """y := f(x)"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_04(self):
+        code = """y0 = y1 := f(x)"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_06(self):
+        code = """((a, b) := (1, 2))"""
+
+        # TODO Cython correctly generates an error but the message could be better
+        with self.assertRaisesRegex(SyntaxError, ""):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_07(self):
+        code = """def spam(a = b := 42): pass"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_08(self):
+        code = """def spam(a: b := 42 = 5): pass"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_09(self):
+        code = """spam(a=b := 'c')"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_10(self):
+        code = """spam(x = y := f(x))"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_11(self):
+        code = """spam(a=1, b := 2)"""
+
+        with self.assertRaisesRegex(SyntaxError,
+            "follow.* keyword arg"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_12(self):
+        code = """spam(a=1, (b := 2))"""
+
+        with self.assertRaisesRegex(SyntaxError,
+            "follow.* keyword arg"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_13(self):
+        code = """spam(a=1, (b := 2))"""
+
+        with self.assertRaisesRegex(SyntaxError,
+            "follow.* keyword arg"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_14(self):
+        code = """(x := lambda: y := 1)"""
+
+        with self.assertRaisesRegex(SyntaxError, "invalid syntax"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_15(self):
+        code = """(lambda: x := 1)"""
+
+        # TODO at the moment the error message is valid, but not the same as Python
+        with self.assertRaisesRegex(SyntaxError,
+            ""):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_16(self):
+        code = "[i + 1 for i in i := [1,2]]"
+
+        # TODO at the moment the error message is valid, but not the same as Python
+        with self.assertRaisesRegex(SyntaxError, ""):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_17(self):
+        code = "[i := 0, j := 1 for i, j in [(1, 2), (3, 4)]]"
+
+        # TODO at the moment the error message is valid, but not the same as Python
+        with self.assertRaisesRegex(SyntaxError, ""):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_in_class_body(self):
+        code = """class Foo():
+            [(42, 1 + ((( j := i )))) for i in range(5)]
+        """
+
+        with self.assertRaisesRegex(SyntaxError,
+            "assignment expression within a comprehension cannot be used in a class body"):
+            exec(code, {}, {})
+
+    def test_named_expression_invalid_rebinding_comprehension_iteration_variable(self):
+        cases = [
+            ("Local reuse", 'i', "[i := 0 for i in range(5)]"),
+            ("Nested reuse", 'j', "[[(j := 0) for i in range(5)] for j in range(5)]"),
+            ("Reuse inner loop target", 'j', "[(j := 0) for i in range(5) for j in range(5)]"),
+            ("Unpacking reuse", 'i', "[i := 0 for i, j in [(0, 1)]]"),
+            ("Reuse in loop condition", 'i', "[i+1 for i in range(5) if (i := 0)]"),
+            ("Unreachable reuse", 'i', "[False or (i:=0) for i in range(5)]"),
+            ("Unreachable nested reuse", 'i',
+                "[(i, j) for i in range(5) for j in range(5) if True or (i:=10)]"),
+        ]
+        for case, target, code in cases:
+            msg = f"assignment expression cannot rebind comprehension iteration variable '{target}'"
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {})
+
+    def test_named_expression_invalid_rebinding_comprehension_inner_loop(self):
+        cases = [
+            ("Inner reuse", 'j', "[i for i in range(5) if (j := 0) for j in range(5)]"),
+            ("Inner unpacking reuse", 'j', "[i for i in range(5) if (j := 0) for j, k in [(0, 1)]]"),
+        ]
+        for case, target, code in cases:
+            msg = f"comprehension inner loop cannot rebind assignment expression target '{target}'"
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}) # Module scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {}) # Class scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(f"lambda: {code}", {}) # Function scope
+
+    def test_named_expression_invalid_comprehension_iterable_expression(self):
+        cases = [
+            ("Top level", "[i for i in (i := range(5))]"),
+            ("Inside tuple", "[i for i in (2, 3, i := range(5))]"),
+            ("Inside list", "[i for i in [2, 3, i := range(5)]]"),
+            ("Different name", "[i for i in (j := range(5))]"),
+            ("Lambda expression", "[i for i in (lambda:(j := range(5)))()]"),
+            ("Inner loop", "[i for i in range(5) for j in (i := range(5))]"),
+            ("Nested comprehension", "[i for i in [j for j in (k := range(5))]]"),
+            ("Nested comprehension condition", "[i for i in [j for j in range(5) if (j := True)]]"),
+            ("Nested comprehension body", "[i for i in [(j := True) for j in range(5)]]"),
+        ]
+        msg = "assignment expression cannot be used in a comprehension iterable expression"
+        for case, code in cases:
+            with self.subTest(case=case):
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}) # Module scope - FIXME this test puts it in __invoke in cython_inline
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(code, {}, {}) # Class scope
+                with self.assertRaisesRegex(SyntaxError, msg):
+                    exec(f"lambda: {code}", {}) # Function scope
+
+
+class NamedExpressionAssignmentTest(unittest.TestCase):
+
+    def test_named_expression_assignment_01(self):
+        (a := 10)
+
+        self.assertEqual(a, 10)
+
+    def test_named_expression_assignment_02(self):
+        a = 20
+        (a := a)
+
+        self.assertEqual(a, 20)
+
+    def test_named_expression_assignment_03(self):
+        (total := 1 + 2)
+
+        self.assertEqual(total, 3)
+
+    def test_named_expression_assignment_04(self):
+        (info := (1, 2, 3))
+
+        self.assertEqual(info, (1, 2, 3))
+
+    def test_named_expression_assignment_05(self):
+        (x := 1, 2)
+
+        self.assertEqual(x, 1)
+
+    def test_named_expression_assignment_06(self):
+        (z := (y := (x := 0)))
+
+        self.assertEqual(x, 0)
+        self.assertEqual(y, 0)
+        self.assertEqual(z, 0)
+
+    def test_named_expression_assignment_07(self):
+        (loc := (1, 2))
+
+        self.assertEqual(loc, (1, 2))
+
+    def test_named_expression_assignment_08(self):
+        if spam := "eggs":
+            self.assertEqual(spam, "eggs")
+        else: self.fail("variable was not assigned using named expression")
+
+    def test_named_expression_assignment_09(self):
+        if True and (spam := True):
+            self.assertTrue(spam)
+        else: self.fail("variable was not assigned using named expression")
+
+    def test_named_expression_assignment_10(self):
+        if (match := 10) == 10:
+            pass
+        else: self.fail("variable was not assigned using named expression")
+
+    def test_named_expression_assignment_11(self):
+        def spam(a):
+            return a
+        input_data = [1, 2, 3]
+        res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0]
+
+        self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)])
+
+    def test_named_expression_assignment_12(self):
+        def spam(a):
+            return a
+        res = [[y := spam(x), x/y] for x in range(1, 5)]
+
+        self.assertEqual(res, [[1, 1.0], [2, 1.0], [3, 1.0], [4, 1.0]])
+
+    def test_named_expression_assignment_13(self):
+        length = len(lines := [1, 2])
+
+        self.assertEqual(length, 2)
+        self.assertEqual(lines, [1,2])
+
+    def test_named_expression_assignment_14(self):
+        """
+        Where all variables are positive integers, and a is at least as large
+        as the n'th root of x, this algorithm returns the floor of the n'th
+        root of x (and roughly doubling the number of accurate bits per
+        iteration):
+        """
+        a = 9
+        n = 2
+        x = 3
+
+        while a > (d := x // a**(n-1)):
+            a = ((n-1)*a + d) // n
+
+        self.assertEqual(a, 1)
+
+    def test_named_expression_assignment_15(self):
+        while a := False:
+            pass  # This will not run
+
+        self.assertEqual(a, False)
+
+    def test_named_expression_assignment_16(self):
+        a, b = 1, 2
+        fib = {(c := a): (a := b) + (b := a + c) - b for __ in range(6)}
+        self.assertEqual(fib, {1: 2, 2: 3, 3: 5, 5: 8, 8: 13, 13: 21})
+
+
+class NamedExpressionScopeTest(unittest.TestCase):
+
+    def test_named_expression_scope_01(self):
+        code = """def spam():
+    (a := 5)
+print(a)"""
+
+        # FIXME for some reason the error message raised is a nonsense filename instead of "undeclared name not builtin"
+        # "name .* not"):
+        with self.assertRaisesRegex(SyntaxError if cython.compiled else NameError, ""):
+            exec(code, {}, {})
+
+    def test_named_expression_scope_02(self):
+        total = 0
+        partial_sums = [total := total + v for v in range(5)]
+
+        self.assertEqual(partial_sums, [0, 1, 3, 6, 10])
+        self.assertEqual(total, 10)
+
+    def test_named_expression_scope_03(self):
+        containsOne = any((lastNum := num) == 1 for num in [1, 2, 3])
+
+        self.assertTrue(containsOne)
+        self.assertEqual(lastNum, 1)
+
+    def test_named_expression_scope_04(self):
+        def spam(a):
+            return a
+        res = [[y := spam(x), x/y] for x in range(1, 5)]
+
+        self.assertEqual(y, 4)
+
+    def test_named_expression_scope_05(self):
+        def spam(a):
+            return a
+        input_data = [1, 2, 3]
+        res = [(x, y, x/y) for x in input_data if (y := spam(x)) > 0]
+
+        self.assertEqual(res, [(1, 1, 1.0), (2, 2, 1.0), (3, 3, 1.0)])
+        self.assertEqual(y, 3)
+
+    def test_named_expression_scope_06(self):
+        res = [[spam := i for i in range(3)] for j in range(2)]
+
+        self.assertEqual(res, [[0, 1, 2], [0, 1, 2]])
+        self.assertEqual(spam, 2)
+
+    def test_named_expression_scope_07(self):
+        len(lines := [1, 2])
+
+        self.assertEqual(lines, [1, 2])
+
+    def test_named_expression_scope_08(self):
+        def spam(a):
+            return a
+
+        def eggs(b):
+            return b * 2
+
+        res = [spam(a := eggs(b := h)) for h in range(2)]
+
+        self.assertEqual(res, [0, 2])
+        self.assertEqual(a, 2)
+        self.assertEqual(b, 1)
+
+    def test_named_expression_scope_09(self):
+        def spam(a):
+            return a
+
+        def eggs(b):
+            return b * 2
+
+        res = [spam(a := eggs(a := h)) for h in range(2)]
+
+        self.assertEqual(res, [0, 2])
+        self.assertEqual(a, 2)
+
+    def test_named_expression_scope_10(self):
+        res = [b := [a := 1 for i in range(2)] for j in range(2)]
+
+        self.assertEqual(res, [[1, 1], [1, 1]])
+        self.assertEqual(a, 1)
+        self.assertEqual(b, [1, 1])
+
+    def test_named_expression_scope_11(self):
+        res = [j := i for i in range(5)]
+
+        self.assertEqual(res, [0, 1, 2, 3, 4])
+        self.assertEqual(j, 4)
+
+    def test_named_expression_scope_17(self):
+        b = 0
+        res = [b := i + b for i in range(5)]
+
+        self.assertEqual(res, [0, 1, 3, 6, 10])
+        self.assertEqual(b, 10)
+
+    def test_named_expression_scope_18(self):
+        def spam(a):
+            return a
+
+        res = spam(b := 2)
+
+        self.assertEqual(res, 2)
+        self.assertEqual(b, 2)
+
+    def test_named_expression_scope_19(self):
+        def spam(a):
+            return a
+
+        res = spam((b := 2))
+
+        self.assertEqual(res, 2)
+        self.assertEqual(b, 2)
+
+    def test_named_expression_scope_20(self):
+        def spam(a):
+            return a
+
+        res = spam(a=(b := 2))
+
+        self.assertEqual(res, 2)
+        self.assertEqual(b, 2)
+
+    def test_named_expression_scope_21(self):
+        def spam(a, b):
+            return a + b
+
+        res = spam(c := 2, b=1)
+
+        self.assertEqual(res, 3)
+        self.assertEqual(c, 2)
+
+    def test_named_expression_scope_22(self):
+        def spam(a, b):
+            return a + b
+
+        res = spam((c := 2), b=1)
+
+        self.assertEqual(res, 3)
+        self.assertEqual(c, 2)
+
+    def test_named_expression_scope_23(self):
+        def spam(a, b):
+            return a + b
+
+        res = spam(b=(c := 2), a=1)
+
+        self.assertEqual(res, 3)
+        self.assertEqual(c, 2)
+
+    def test_named_expression_scope_24(self):
+        a = 10
+        def spam():
+            nonlocal a
+            (a := 20)
+        spam()
+
+        self.assertEqual(a, 20)
+
+    def test_named_expression_scope_25(self):
+        ns = {}
+        code = """a = 10
+def spam():
+    global a
+    (a := 20)
+spam()"""
+
+        exec(code, ns, {})
+
+        self.assertEqual(ns["a"], 20)
+
+    def test_named_expression_variable_reuse_in_comprehensions(self):
+        # The compiler is expected to raise syntax error for comprehension
+        # iteration variables, but should be fine with rebinding of other
+        # names (e.g. globals, nonlocals, other assignment expressions)
+
+        # The cases are all defined to produce the same expected result
+        # Each comprehension is checked at both function scope and module scope
+        rebinding = "[x := i for i in range(3) if (x := i) or not x]"
+        filter_ref = "[x := i for i in range(3) if x or not x]"
+        body_ref = "[x for i in range(3) if (x := i) or not x]"
+        nested_ref = "[j for i in range(3) if x or not x for j in range(3) if (x := i)][:-3]"
+        cases = [
+            ("Rebind global", f"x = 1; result = {rebinding}"),
+            ("Rebind nonlocal", f"result, x = (lambda x=1: ({rebinding}, x))()"),
+            ("Filter global", f"x = 1; result = {filter_ref}"),
+            ("Filter nonlocal", f"result, x = (lambda x=1: ({filter_ref}, x))()"),
+            ("Body global", f"x = 1; result = {body_ref}"),
+            ("Body nonlocal", f"result, x = (lambda x=1: ({body_ref}, x))()"),
+            ("Nested global", f"x = 1; result = {nested_ref}"),
+            ("Nested nonlocal", f"result, x = (lambda x=1: ({nested_ref}, x))()"),
+        ]
+        for case, code in cases:
+            with self.subTest(case=case):
+                ns = {}
+                exec(code, ns)
+                self.assertEqual(ns["x"], 2)
+                self.assertEqual(ns["result"], [0, 1, 2])
+
+if __name__ == "__main__":
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_subclassinit.py cython-0.20.1+1~202203241016-9537/tests/run/test_subclassinit.py
--- cython-0.20.1+1~201611251650-6686/tests/run/test_subclassinit.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_subclassinit.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,316 @@
+# mode: run
+# tag: pure3.6
+# cython: language_level=3str
+
+import sys
+HAS_NATIVE_SUPPORT = sys.version_info >= (3, 6)
+IS_PY2 = sys.version_info[0] == 2
+
+import re
+import types
+import unittest
+
+ZERO = 0
+
+skip_if_not_native = unittest.skipIf(not HAS_NATIVE_SUPPORT, "currently requires Python 3.6+")
+
+
+class Test(unittest.TestCase):
+    if not hasattr(unittest.TestCase, 'assertRegex'):
+        def assertRegex(self, value, regex):
+            self.assertTrue(re.search(regex, str(value)),
+                            "'%s' did not match '%s'" % (value, regex))
+
+    if not hasattr(unittest.TestCase, 'assertCountEqual'):
+        def assertCountEqual(self, first, second):
+            self.assertEqual(set(first), set(second))
+            self.assertEqual(len(first), len(second))
+
+    def test_init_subclass(self):
+        class A:
+            initialized = False
+
+            def __init_subclass__(cls):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__()
+                cls.initialized = True
+
+        class B(A):
+            pass
+
+        self.assertFalse(A.initialized)
+        self.assertTrue(B.initialized)
+
+    def test_init_subclass_dict(self):
+        class A(dict):
+            initialized = False
+
+            def __init_subclass__(cls):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__()
+                cls.initialized = True
+
+        class B(A):
+            pass
+
+        self.assertFalse(A.initialized)
+        self.assertTrue(B.initialized)
+
+    def test_init_subclass_kwargs(self):
+        class A:
+            def __init_subclass__(cls, **kwargs):
+                cls.kwargs = kwargs
+
+        class B(A, x=3):
+            pass
+
+        self.assertEqual(B.kwargs, dict(x=3))
+
+    def test_init_subclass_error(self):
+        class A:
+            def __init_subclass__(cls):
+                raise RuntimeError
+
+        with self.assertRaises(RuntimeError):
+            class B(A):
+                pass
+
+    def test_init_subclass_wrong(self):
+        class A:
+            def __init_subclass__(cls, whatever):
+                pass
+
+        with self.assertRaises(TypeError):
+            class B(A):
+                pass
+
+    def test_init_subclass_skipped(self):
+        class BaseWithInit:
+            def __init_subclass__(cls, **kwargs):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__(**kwargs)
+                cls.initialized = cls
+
+        class BaseWithoutInit(BaseWithInit):
+            pass
+
+        class A(BaseWithoutInit):
+            pass
+
+        self.assertIs(A.initialized, A)
+        self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit)
+
+    def test_init_subclass_diamond(self):
+        class Base:
+            def __init_subclass__(cls, **kwargs):
+                if HAS_NATIVE_SUPPORT:
+                    super().__init_subclass__(**kwargs)
+                cls.calls = []
+
+        class Left(Base):
+            pass
+
+        class Middle:
+            def __init_subclass__(cls, middle, **kwargs):
+                super().__init_subclass__(**kwargs)
+                cls.calls += [middle]
+
+        class Right(Base):
+            def __init_subclass__(cls, right="right", **kwargs):
+                super().__init_subclass__(**kwargs)
+                cls.calls += [right]
+
+        class A(Left, Middle, Right, middle="middle"):
+            pass
+
+        self.assertEqual(A.calls, ["right", "middle"])
+        self.assertEqual(Left.calls, [])
+        self.assertEqual(Right.calls, [])
+
+    def test_set_name(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class A:
+            d = Descriptor()
+
+        self.assertEqual(A.d.name, "d")
+        self.assertIs(A.d.owner, A)
+
+    @skip_if_not_native
+    def test_set_name_metaclass(self):
+        class Meta(type):
+            def __new__(cls, name, bases, ns):
+                ret = super().__new__(cls, name, bases, ns)
+                self.assertEqual(ret.d.name, "d")
+                self.assertIs(ret.d.owner, ret)
+                return 0
+
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class A(metaclass=Meta):
+            d = Descriptor()
+        self.assertEqual(A, 0)
+
+    def test_set_name_error(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                1 / ZERO
+
+        with self.assertRaises(RuntimeError) as cm:
+            class NotGoingToWork:
+                attr = Descriptor()
+
+        exc = cm.exception
+        self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+        self.assertRegex(str(exc), r'\battr\b')
+        self.assertRegex(str(exc), r'\bDescriptor\b')
+        if HAS_NATIVE_SUPPORT:
+            self.assertIsInstance(exc.__cause__, ZeroDivisionError)
+
+    def test_set_name_wrong(self):
+        class Descriptor:
+            def __set_name__(self):
+                pass
+
+        with self.assertRaises(RuntimeError) as cm:
+            class NotGoingToWork:
+                attr = Descriptor()
+
+        exc = cm.exception
+        self.assertRegex(str(exc), r'\bNotGoingToWork\b')
+        self.assertRegex(str(exc), r'\battr\b')
+        self.assertRegex(str(exc), r'\bDescriptor\b')
+        if HAS_NATIVE_SUPPORT:
+            self.assertIsInstance(exc.__cause__, TypeError)
+
+    def test_set_name_lookup(self):
+        resolved = []
+        class NonDescriptor:
+            def __getattr__(self, name):
+                resolved.append(name)
+
+        class A:
+            d = NonDescriptor()
+
+        self.assertNotIn('__set_name__', resolved,
+                         '__set_name__ is looked up in instance dict')
+
+    @skip_if_not_native
+    def test_set_name_init_subclass(self):
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                self.owner = owner
+                self.name = name
+
+        class Meta(type):
+            def __new__(cls, name, bases, ns):
+                self = super().__new__(cls, name, bases, ns)
+                self.meta_owner = self.owner
+                self.meta_name = self.name
+                return self
+
+        class A:
+            def __init_subclass__(cls):
+                cls.owner = cls.d.owner
+                cls.name = cls.d.name
+
+        class B(A, metaclass=Meta):
+            d = Descriptor()
+
+        self.assertIs(B.owner, B)
+        self.assertEqual(B.name, 'd')
+        self.assertIs(B.meta_owner, B)
+        self.assertEqual(B.name, 'd')
+
+    def test_set_name_modifying_dict(self):
+        notified = []
+        class Descriptor:
+            def __set_name__(self, owner, name):
+                setattr(owner, name + 'x', None)
+                notified.append(name)
+
+        class A:
+            a = Descriptor()
+            b = Descriptor()
+            c = Descriptor()
+            d = Descriptor()
+            e = Descriptor()
+
+        self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e'])
+
+    def test_errors(self):
+        class MyMeta(type):
+            pass
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta, otherarg=1):
+                pass
+
+        if not IS_PY2:
+            with self.assertRaises(TypeError):
+                types.new_class("MyClass", (object,),
+                                dict(metaclass=MyMeta, otherarg=1))
+            types.prepare_class("MyClass", (object,),
+                                dict(metaclass=MyMeta, otherarg=1))
+
+        class MyMeta(type):
+            def __init__(self, name, bases, namespace, otherarg):
+                super().__init__(name, bases, namespace)
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta, otherarg=1):
+                pass
+
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace, otherarg):
+                return super().__new__(cls, name, bases, namespace)
+
+            def __init__(self, name, bases, namespace, otherarg):
+                super().__init__(name, bases, namespace)
+                self.otherarg = otherarg
+
+        class MyClass(metaclass=MyMeta, otherarg=1):
+            pass
+
+        self.assertEqual(MyClass.otherarg, 1)
+
+    @skip_if_not_native
+    def test_errors_changed_pep487(self):
+        # These tests failed before Python 3.6, PEP 487
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace):
+                return super().__new__(cls, name=name, bases=bases,
+                                       dict=namespace)
+
+        with self.assertRaises(TypeError):
+            class MyClass(metaclass=MyMeta):
+                pass
+
+        class MyMeta(type):
+            def __new__(cls, name, bases, namespace, otherarg):
+                self = super().__new__(cls, name, bases, namespace)
+                self.otherarg = otherarg
+                return self
+
+        class MyClass(metaclass=MyMeta, otherarg=1):
+            pass
+
+        self.assertEqual(MyClass.otherarg, 1)
+
+    def test_type(self):
+        t = type('NewClass', (object,), {})
+        self.assertIsInstance(t, type)
+        self.assertEqual(t.__name__, 'NewClass')
+
+        with self.assertRaises(TypeError):
+            type(name='NewClass', bases=(object,), dict={})
+
+
+if __name__ == "__main__":
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_unicode.pyx cython-0.20.1+1~202203241016-9537/tests/run/test_unicode.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/test_unicode.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_unicode.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,2970 @@
+# cython: language_level=3
+
+""" Test script for the Unicode implementation.
+
+Written by Marc-Andre Lemburg (mal@lemburg.com).
+
+(c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
+
+"""
+#import _string
+import codecs
+import itertools
+import operator
+#import struct
+#import sys
+#import unittest
+import warnings
+# from test import support, string_tests
+from contextlib import contextmanager
+
+
+class support(object):
+    @staticmethod
+    def _ignore(func):
+        return unittest.skip("Ignoring CPython-only test")(func)
+
+    def run_with_locale(*args):
+        return support._ignore
+
+    cpython_only = _ignore
+
+    def check_free_after_iterating(*args):
+        pass
+
+    @contextmanager
+    def check_warnings(*args):
+        yield  # ignore any warnings
+
+support = support()
+
+include "test_unicode_string_tests.pxi"
+
+
+############### ORIGINAL TESTS START HERE #################
+
+
+# Error handling (bad decoder return)
+def search_function(encoding):
+    def decode1(input, errors="strict"):
+        return 42 # not a tuple
+    def encode1(input, errors="strict"):
+        return 42 # not a tuple
+    def encode2(input, errors="strict"):
+        return (42, 42) # no unicode
+    def decode2(input, errors="strict"):
+        return (42, 42) # no unicode
+    if encoding=="test.unicode1":
+        return (encode1, decode1, None, None)
+    elif encoding=="test.unicode2":
+        return (encode2, decode2, None, None)
+    else:
+        return None
+codecs.register(search_function)
+
+def duplicate_string(text):
+    """
+    Try to get a fresh clone of the specified text:
+    new object with a reference count of 1.
+
+    This is a best-effort: latin1 single letters and the empty
+    string ('') are singletons and cannot be cloned.
+    """
+    return text.encode().decode()
+
+class StrSubclass(str):
+    pass
+
+class UnicodeTest(CommonTest,
+        MixinStrUnicodeUserStringTest,
+        MixinStrUnicodeTest,
+        unittest.TestCase):
+
+    type2test = str
+
+    def checkequalnofix(self, result, object, methodname, *args):
+        method = getattr(object, methodname)
+        realresult = method(*args)
+        self.assertEqual(realresult, result)
+        self.assertTrue(type(realresult) is type(result))
+
+        # if the original is returned make sure that
+        # this doesn't happen with subclasses
+        if realresult is object:
+            class usub(str):
+                def __repr__(self):
+                    return 'usub(%r)' % str.__repr__(self)
+            object = usub(object)
+            method = getattr(object, methodname)
+            realresult = method(*args)
+            self.assertEqual(realresult, result)
+            self.assertTrue(object is not realresult)
+
+    def test_literals(self):
+        self.assertEqual('\xff', '\u00ff')
+        self.assertEqual('\uffff', '\U0000ffff')
+        self.assertRaises(SyntaxError, eval, '\'\\Ufffffffe\'')
+        self.assertRaises(SyntaxError, eval, '\'\\Uffffffff\'')
+        self.assertRaises(SyntaxError, eval, '\'\\U%08x\'' % 0x110000)
+        # raw strings should not have unicode escapes
+        self.assertNotEqual(r"\u0020", " ")
+
+    def test_ascii(self):
+        if not sys.platform.startswith('java'):
+            # Test basic sanity of repr()
+            self.assertEqual(ascii('abc'), "'abc'")
+            self.assertEqual(ascii('ab\\c'), "'ab\\\\c'")
+            self.assertEqual(ascii('ab\\'), "'ab\\\\'")
+            self.assertEqual(ascii('\\c'), "'\\\\c'")
+            self.assertEqual(ascii('\\'), "'\\\\'")
+            self.assertEqual(ascii('\n'), "'\\n'")
+            self.assertEqual(ascii('\r'), "'\\r'")
+            self.assertEqual(ascii('\t'), "'\\t'")
+            self.assertEqual(ascii('\b'), "'\\x08'")
+            self.assertEqual(ascii("'\""), """'\\'"'""")
+            self.assertEqual(ascii("'\""), """'\\'"'""")
+            self.assertEqual(ascii("'"), '''"'"''')
+            self.assertEqual(ascii('"'), """'"'""")
+            latin1repr = (
+                "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r"
+                "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a"
+                "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI"
+                "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f"
+                "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d"
+                "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b"
+                "\\x9c\\x9d\\x9e\\x9f\\xa0\\xa1\\xa2\\xa3\\xa4\\xa5\\xa6\\xa7\\xa8\\xa9"
+                "\\xaa\\xab\\xac\\xad\\xae\\xaf\\xb0\\xb1\\xb2\\xb3\\xb4\\xb5\\xb6\\xb7"
+                "\\xb8\\xb9\\xba\\xbb\\xbc\\xbd\\xbe\\xbf\\xc0\\xc1\\xc2\\xc3\\xc4\\xc5"
+                "\\xc6\\xc7\\xc8\\xc9\\xca\\xcb\\xcc\\xcd\\xce\\xcf\\xd0\\xd1\\xd2\\xd3"
+                "\\xd4\\xd5\\xd6\\xd7\\xd8\\xd9\\xda\\xdb\\xdc\\xdd\\xde\\xdf\\xe0\\xe1"
+                "\\xe2\\xe3\\xe4\\xe5\\xe6\\xe7\\xe8\\xe9\\xea\\xeb\\xec\\xed\\xee\\xef"
+                "\\xf0\\xf1\\xf2\\xf3\\xf4\\xf5\\xf6\\xf7\\xf8\\xf9\\xfa\\xfb\\xfc\\xfd"
+                "\\xfe\\xff'")
+            testrepr = ascii(''.join(map(chr, range(256))))
+            self.assertEqual(testrepr, latin1repr)
+            # Test ascii works on wide unicode escapes without overflow.
+            self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096),
+                             ascii("\U00010000" * 39 + "\uffff" * 4096))
+
+            class WrongRepr:
+                def __repr__(self):
+                    return b'byte-repr'
+            self.assertRaises(TypeError, ascii, WrongRepr())
+
+    def test_repr(self):
+        if not sys.platform.startswith('java'):
+            # Test basic sanity of repr()
+            self.assertEqual(repr('abc'), "'abc'")
+            self.assertEqual(repr('ab\\c'), "'ab\\\\c'")
+            self.assertEqual(repr('ab\\'), "'ab\\\\'")
+            self.assertEqual(repr('\\c'), "'\\\\c'")
+            self.assertEqual(repr('\\'), "'\\\\'")
+            self.assertEqual(repr('\n'), "'\\n'")
+            self.assertEqual(repr('\r'), "'\\r'")
+            self.assertEqual(repr('\t'), "'\\t'")
+            self.assertEqual(repr('\b'), "'\\x08'")
+            self.assertEqual(repr("'\""), """'\\'"'""")
+            self.assertEqual(repr("'\""), """'\\'"'""")
+            self.assertEqual(repr("'"), '''"'"''')
+            self.assertEqual(repr('"'), """'"'""")
+            latin1repr = (
+                "'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b\\x0c\\r"
+                "\\x0e\\x0f\\x10\\x11\\x12\\x13\\x14\\x15\\x16\\x17\\x18\\x19\\x1a"
+                "\\x1b\\x1c\\x1d\\x1e\\x1f !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHI"
+                "JKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\x7f"
+                "\\x80\\x81\\x82\\x83\\x84\\x85\\x86\\x87\\x88\\x89\\x8a\\x8b\\x8c\\x8d"
+                "\\x8e\\x8f\\x90\\x91\\x92\\x93\\x94\\x95\\x96\\x97\\x98\\x99\\x9a\\x9b"
+                "\\x9c\\x9d\\x9e\\x9f\\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9"
+                "\xaa\xab\xac\\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7"
+                "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5"
+                "\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3"
+                "\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1"
+                "\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
+                "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd"
+                "\xfe\xff'")
+            testrepr = repr(''.join(map(chr, range(256))))
+            self.assertEqual(testrepr, latin1repr)
+            # Test repr works on wide unicode escapes without overflow.
+            self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096),
+                             repr("\U00010000" * 39 + "\uffff" * 4096))
+
+            class WrongRepr:
+                def __repr__(self):
+                    return b'byte-repr'
+            self.assertRaises(TypeError, repr, WrongRepr())
+
+    def test_iterators(self):
+        # Make sure unicode objects have an __iter__ method
+        it = "\u1111\u2222\u3333".__iter__()
+        self.assertEqual(next(it), "\u1111")
+        self.assertEqual(next(it), "\u2222")
+        self.assertEqual(next(it), "\u3333")
+        self.assertRaises(StopIteration, next, it)
+
+    def test_count(self):
+        CommonTest.test_count(self)
+        # check mixed argument types
+        self.checkequalnofix(3,  'aaa', 'count', 'a')
+        self.checkequalnofix(0,  'aaa', 'count', 'b')
+        self.checkequalnofix(3, 'aaa', 'count',  'a')
+        self.checkequalnofix(0, 'aaa', 'count',  'b')
+        self.checkequalnofix(0, 'aaa', 'count',  'b')
+        self.checkequalnofix(1, 'aaa', 'count',  'a', -1)
+        self.checkequalnofix(3, 'aaa', 'count',  'a', -10)
+        self.checkequalnofix(2, 'aaa', 'count',  'a', 0, -1)
+        self.checkequalnofix(0, 'aaa', 'count',  'a', 0, -10)
+        # test mixed kinds
+        self.checkequal(10, '\u0102' + 'a' * 10, 'count', 'a')
+        self.checkequal(10, '\U00100304' + 'a' * 10, 'count', 'a')
+        self.checkequal(10, '\U00100304' + '\u0102' * 10, 'count', '\u0102')
+        self.checkequal(0, 'a' * 10, 'count', '\u0102')
+        self.checkequal(0, 'a' * 10, 'count', '\U00100304')
+        self.checkequal(0, '\u0102' * 10, 'count', '\U00100304')
+        self.checkequal(10, '\u0102' + 'a_' * 10, 'count', 'a_')
+        self.checkequal(10, '\U00100304' + 'a_' * 10, 'count', 'a_')
+        self.checkequal(10, '\U00100304' + '\u0102_' * 10, 'count', '\u0102_')
+        self.checkequal(0, 'a' * 10, 'count', 'a\u0102')
+        self.checkequal(0, 'a' * 10, 'count', 'a\U00100304')
+        self.checkequal(0, '\u0102' * 10, 'count', '\u0102\U00100304')
+
+    def test_find(self):
+        CommonTest.test_find(self)
+        # test implementation details of the memchr fast path
+        self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0120')
+        self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0220')
+        self.checkequal(100, 'a' * 100 + '\U00100304', 'find', '\U00100304')
+        self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00100204')
+        self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00102004')
+        # check mixed argument types
+        self.checkequalnofix(0,  'abcdefghiabc', 'find', 'abc')
+        self.checkequalnofix(9,  'abcdefghiabc', 'find', 'abc', 1)
+        self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4)
+
+        self.assertRaises(TypeError, 'hello'.find)
+        self.assertRaises(TypeError, 'hello'.find, 42)
+        # test mixed kinds
+        self.checkequal(100, '\u0102' * 100 + 'a', 'find', 'a')
+        self.checkequal(100, '\U00100304' * 100 + 'a', 'find', 'a')
+        self.checkequal(100, '\U00100304' * 100 + '\u0102', 'find', '\u0102')
+        self.checkequal(-1, 'a' * 100, 'find', '\u0102')
+        self.checkequal(-1, 'a' * 100, 'find', '\U00100304')
+        self.checkequal(-1, '\u0102' * 100, 'find', '\U00100304')
+        self.checkequal(100, '\u0102' * 100 + 'a_', 'find', 'a_')
+        self.checkequal(100, '\U00100304' * 100 + 'a_', 'find', 'a_')
+        self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'find', '\u0102_')
+        self.checkequal(-1, 'a' * 100, 'find', 'a\u0102')
+        self.checkequal(-1, 'a' * 100, 'find', 'a\U00100304')
+        self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304')
+
+    def test_rfind(self):
+        CommonTest.test_rfind(self)
+        # test implementation details of the memrchr fast path
+        self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0120')
+        self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0220')
+        self.checkequal(0, '\U00100304' + 'a' * 100, 'rfind', '\U00100304')
+        self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00100204')
+        self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00102004')
+        # check mixed argument types
+        self.checkequalnofix(9,   'abcdefghiabc', 'rfind', 'abc')
+        self.checkequalnofix(12,  'abcdefghiabc', 'rfind', '')
+        self.checkequalnofix(12, 'abcdefghiabc', 'rfind',  '')
+        # test mixed kinds
+        self.checkequal(0, 'a' + '\u0102' * 100, 'rfind', 'a')
+        self.checkequal(0, 'a' + '\U00100304' * 100, 'rfind', 'a')
+        self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rfind', '\u0102')
+        self.checkequal(-1, 'a' * 100, 'rfind', '\u0102')
+        self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304')
+        self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304')
+        self.checkequal(0, '_a' + '\u0102' * 100, 'rfind', '_a')
+        self.checkequal(0, '_a' + '\U00100304' * 100, 'rfind', '_a')
+        self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rfind', '_\u0102')
+        self.checkequal(-1, 'a' * 100, 'rfind', '\u0102a')
+        self.checkequal(-1, 'a' * 100, 'rfind', '\U00100304a')
+        self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102')
+
+    def test_index(self):
+        CommonTest.test_index(self)
+        self.checkequalnofix(0, 'abcdefghiabc', 'index',  '')
+        self.checkequalnofix(3, 'abcdefghiabc', 'index',  'def')
+        self.checkequalnofix(0, 'abcdefghiabc', 'index',  'abc')
+        self.checkequalnofix(9, 'abcdefghiabc', 'index',  'abc', 1)
+        self.assertRaises(ValueError, 'abcdefghiabc'.index, 'hib')
+        self.assertRaises(ValueError, 'abcdefghiab'.index,  'abc', 1)
+        self.assertRaises(ValueError, 'abcdefghi'.index,  'ghi', 8)
+        self.assertRaises(ValueError, 'abcdefghi'.index,  'ghi', -1)
+        # test mixed kinds
+        self.checkequal(100, '\u0102' * 100 + 'a', 'index', 'a')
+        self.checkequal(100, '\U00100304' * 100 + 'a', 'index', 'a')
+        self.checkequal(100, '\U00100304' * 100 + '\u0102', 'index', '\u0102')
+        self.assertRaises(ValueError, ('a' * 100).index, '\u0102')
+        self.assertRaises(ValueError, ('a' * 100).index, '\U00100304')
+        self.assertRaises(ValueError, ('\u0102' * 100).index, '\U00100304')
+        self.checkequal(100, '\u0102' * 100 + 'a_', 'index', 'a_')
+        self.checkequal(100, '\U00100304' * 100 + 'a_', 'index', 'a_')
+        self.checkequal(100, '\U00100304' * 100 + '\u0102_', 'index', '\u0102_')
+        self.assertRaises(ValueError, ('a' * 100).index, 'a\u0102')
+        self.assertRaises(ValueError, ('a' * 100).index, 'a\U00100304')
+        self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304')
+
+    def test_rindex(self):
+        CommonTest.test_rindex(self)
+        self.checkequalnofix(12, 'abcdefghiabc', 'rindex',  '')
+        self.checkequalnofix(3,  'abcdefghiabc', 'rindex',  'def')
+        self.checkequalnofix(9,  'abcdefghiabc', 'rindex',  'abc')
+        self.checkequalnofix(0,  'abcdefghiabc', 'rindex',  'abc', 0, -1)
+
+        self.assertRaises(ValueError, 'abcdefghiabc'.rindex,  'hib')
+        self.assertRaises(ValueError, 'defghiabc'.rindex,  'def', 1)
+        self.assertRaises(ValueError, 'defghiabc'.rindex,  'abc', 0, -1)
+        self.assertRaises(ValueError, 'abcdefghi'.rindex,  'ghi', 0, 8)
+        self.assertRaises(ValueError, 'abcdefghi'.rindex,  'ghi', 0, -1)
+        # test mixed kinds
+        self.checkequal(0, 'a' + '\u0102' * 100, 'rindex', 'a')
+        self.checkequal(0, 'a' + '\U00100304' * 100, 'rindex', 'a')
+        self.checkequal(0, '\u0102' + '\U00100304' * 100, 'rindex', '\u0102')
+        self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102')
+        self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304')
+        self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304')
+        self.checkequal(0, '_a' + '\u0102' * 100, 'rindex', '_a')
+        self.checkequal(0, '_a' + '\U00100304' * 100, 'rindex', '_a')
+        self.checkequal(0, '_\u0102' + '\U00100304' * 100, 'rindex', '_\u0102')
+        self.assertRaises(ValueError, ('a' * 100).rindex, '\u0102a')
+        self.assertRaises(ValueError, ('a' * 100).rindex, '\U00100304a')
+        self.assertRaises(ValueError, ('\u0102' * 100).rindex, '\U00100304\u0102')
+
+    @unittest.skipIf(sys.version_info < (3, 6), 'Python str.translate() test requires Py3.6+')
+    def test_maketrans_translate(self):
+        # these work with plain translate()
+        self.checkequalnofix('bbbc', 'abababc', 'translate',
+                             {ord('a'): None})
+        self.checkequalnofix('iiic', 'abababc', 'translate',
+                             {ord('a'): None, ord('b'): ord('i')})
+        self.checkequalnofix('iiix', 'abababc', 'translate',
+                             {ord('a'): None, ord('b'): ord('i'), ord('c'): 'x'})
+        self.checkequalnofix('c', 'abababc', 'translate',
+                             {ord('a'): None, ord('b'): ''})
+        self.checkequalnofix('xyyx', 'xzx', 'translate',
+                             {ord('z'): 'yy'})
+
+        # this needs maketrans()
+        self.checkequalnofix('abababc', 'abababc', 'translate',
+                             {'b': ''})
+        tbl = self.type2test.maketrans({'a': None, 'b': ''})
+        self.checkequalnofix('c', 'abababc', 'translate', tbl)
+        # test alternative way of calling maketrans()
+        tbl = self.type2test.maketrans('abc', 'xyz', 'd')
+        self.checkequalnofix('xyzzy', 'abdcdcbdddd', 'translate', tbl)
+
+        # various tests switching from ASCII to latin1 or the opposite;
+        # same length, remove a letter, or replace with a longer string.
+        self.assertEqual("[a]".translate(str.maketrans('a', 'X')),
+                         "[X]")
+        self.assertEqual("[a]".translate(str.maketrans({'a': 'X'})),
+                         "[X]")
+        self.assertEqual("[a]".translate(str.maketrans({'a': None})),
+                         "[]")
+        self.assertEqual("[a]".translate(str.maketrans({'a': 'XXX'})),
+                         "[XXX]")
+        self.assertEqual("[a]".translate(str.maketrans({'a': '\xe9'})),
+                         "[\xe9]")
+        self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '123'})),
+                         "x123")
+        self.assertEqual('axb'.translate(str.maketrans({'a': None, 'b': '\xe9'})),
+                         "x\xe9")
+
+        # test non-ASCII (don't take the fast-path)
+        self.assertEqual("[a]".translate(str.maketrans({'a': '<\xe9>'})),
+                         "[<\xe9>]")
+        self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': 'a'})),
+                         "[a]")
+        self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': None})),
+                         "[]")
+        self.assertEqual("[\xe9]".translate(str.maketrans({'\xe9': '123'})),
+                         "[123]")
+        self.assertEqual("[a\xe9]".translate(str.maketrans({'a': '<\u20ac>'})),
+                         "[<\u20ac>\xe9]")
+
+        # invalid Unicode characters
+        invalid_char = 0x10ffff+1
+        for before in "a\xe9\u20ac\U0010ffff":
+            mapping = str.maketrans({before: invalid_char})
+            text = "[%s]" % before
+            self.assertRaises(ValueError, text.translate, mapping)
+
+        # errors
+        self.assertRaises(TypeError, self.type2test.maketrans)
+        self.assertRaises(ValueError, self.type2test.maketrans, 'abc', 'defg')
+        self.assertRaises(TypeError, self.type2test.maketrans, 2, 'def')
+        self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 2)
+        self.assertRaises(TypeError, self.type2test.maketrans, 'abc', 'def', 2)
+        self.assertRaises(ValueError, self.type2test.maketrans, {'xy': 2})
+        self.assertRaises(TypeError, self.type2test.maketrans, {(1,): 2})
+
+        self.assertRaises(TypeError, 'hello'.translate)
+        self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz')
+
+    def test_split(self):
+        CommonTest.test_split(self)
+
+        # test mixed kinds
+        for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'):
+            left *= 9
+            right *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                self.checkequal([left + right],
+                                left + right, 'split', delim)
+                self.checkequal([left, right],
+                                left + delim + right, 'split', delim)
+                self.checkequal([left + right],
+                                left + right, 'split', delim * 2)
+                self.checkequal([left, right],
+                                left + delim * 2 + right, 'split', delim *2)
+
+    def test_rsplit(self):
+        CommonTest.test_rsplit(self)
+        # test mixed kinds
+        for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'):
+            left *= 9
+            right *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                self.checkequal([left + right],
+                                left + right, 'rsplit', delim)
+                self.checkequal([left, right],
+                                left + delim + right, 'rsplit', delim)
+                self.checkequal([left + right],
+                                left + right, 'rsplit', delim * 2)
+                self.checkequal([left, right],
+                                left + delim * 2 + right, 'rsplit', delim *2)
+
+    def test_partition(self):
+        MixinStrUnicodeUserStringTest.test_partition(self)
+        # test mixed kinds
+        self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200')
+        for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'):
+            left *= 9
+            right *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                self.checkequal((left + right, '', ''),
+                                left + right, 'partition', delim)
+                self.checkequal((left, delim, right),
+                                left + delim + right, 'partition', delim)
+                self.checkequal((left + right, '', ''),
+                                left + right, 'partition', delim * 2)
+                self.checkequal((left, delim * 2, right),
+                                left + delim * 2 + right, 'partition', delim * 2)
+
+    def test_rpartition(self):
+        MixinStrUnicodeUserStringTest.test_rpartition(self)
+        # test mixed kinds
+        self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200')
+        for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'):
+            left *= 9
+            right *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                self.checkequal(('', '', left + right),
+                                left + right, 'rpartition', delim)
+                self.checkequal((left, delim, right),
+                                left + delim + right, 'rpartition', delim)
+                self.checkequal(('', '', left + right),
+                                left + right, 'rpartition', delim * 2)
+                self.checkequal((left, delim * 2, right),
+                                left + delim * 2 + right, 'rpartition', delim * 2)
+
+    def test_join(self):
+        MixinStrUnicodeUserStringTest.test_join(self)
+
+        class MyWrapper:
+            def __init__(self, sval): self.sval = sval
+            def __str__(self): return self.sval
+
+        # mixed arguments
+        self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
+        self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd'))
+        self.checkequalnofix('w x y z', ' ', 'join', Sequence('wxyz'))
+        self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
+        self.checkequalnofix('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
+        self.checkequalnofix('abcd', '', 'join', ('a', 'b', 'c', 'd'))
+        self.checkequalnofix('w x y z', ' ', 'join', Sequence('wxyz'))
+        self.checkraises(TypeError, ' ', 'join', ['1', '2', MyWrapper('foo')])
+        self.checkraises(TypeError, ' ', 'join', ['1', '2', '3', bytes()])
+        self.checkraises(TypeError, ' ', 'join', [1, 2, 3])
+        self.checkraises(TypeError, ' ', 'join', ['1', '2', 3])
+
+    @unittest.skipIf(sys.maxsize > 2**32,
+        'needs too much memory on a 64-bit platform')
+    def test_join_overflow(self):
+        size = int(sys.maxsize**0.5) + 1
+        seq = ('A' * size,) * size
+        self.assertRaises(OverflowError, ''.join, seq)
+
+    def test_replace(self):
+        CommonTest.test_replace(self)
+
+        # method call forwarded from str implementation because of unicode argument
+        self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1)
+        self.assertRaises(TypeError, 'replace'.replace, "r", 42)
+        # test mixed kinds
+        for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'):
+            left *= 9
+            right *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                for repl in ('d', '\u0103', '\U00010303'):
+                    self.checkequal(left + right,
+                                    left + right, 'replace', delim, repl)
+                    self.checkequal(left + repl + right,
+                                    left + delim + right,
+                                    'replace', delim, repl)
+                    self.checkequal(left + right,
+                                    left + right, 'replace', delim * 2, repl)
+                    self.checkequal(left + repl + right,
+                                    left + delim * 2 + right,
+                                    'replace', delim * 2, repl)
+
+    @support.cpython_only
+    def test_replace_id(self):
+        pattern = 'abc'
+        text = 'abc def'
+        self.assertIs(text.replace(pattern, pattern), text)
+
+    def test_bytes_comparison(self):
+        with support.check_warnings():
+            warnings.simplefilter('ignore', BytesWarning)
+            self.assertEqual('abc' == b'abc', False)
+            self.assertEqual('abc' != b'abc', True)
+            self.assertEqual('abc' == bytearray(b'abc'), False)
+            self.assertEqual('abc' != bytearray(b'abc'), True)
+
+    def test_comparison(self):
+        # Comparisons:
+        self.assertEqual('abc', 'abc')
+        self.assertTrue('abcd' > 'abc')
+        self.assertTrue('abc' < 'abcd')
+
+        if 0:
+            # Move these tests to a Unicode collation module test...
+            # Testing UTF-16 code point order comparisons...
+
+            # No surrogates, no fixup required.
+            self.assertTrue('\u0061' < '\u20ac')
+            # Non surrogate below surrogate value, no fixup required
+            self.assertTrue('\u0061' < '\ud800\udc02')
+
+            # Non surrogate above surrogate value, fixup required
+            def test_lecmp(s, s2):
+                self.assertTrue(s < s2)
+
+            def test_fixup(s):
+                s2 = '\ud800\udc01'
+                test_lecmp(s, s2)
+                s2 = '\ud900\udc01'
+                test_lecmp(s, s2)
+                s2 = '\uda00\udc01'
+                test_lecmp(s, s2)
+                s2 = '\udb00\udc01'
+                test_lecmp(s, s2)
+                s2 = '\ud800\udd01'
+                test_lecmp(s, s2)
+                s2 = '\ud900\udd01'
+                test_lecmp(s, s2)
+                s2 = '\uda00\udd01'
+                test_lecmp(s, s2)
+                s2 = '\udb00\udd01'
+                test_lecmp(s, s2)
+                s2 = '\ud800\ude01'
+                test_lecmp(s, s2)
+                s2 = '\ud900\ude01'
+                test_lecmp(s, s2)
+                s2 = '\uda00\ude01'
+                test_lecmp(s, s2)
+                s2 = '\udb00\ude01'
+                test_lecmp(s, s2)
+                s2 = '\ud800\udfff'
+                test_lecmp(s, s2)
+                s2 = '\ud900\udfff'
+                test_lecmp(s, s2)
+                s2 = '\uda00\udfff'
+                test_lecmp(s, s2)
+                s2 = '\udb00\udfff'
+                test_lecmp(s, s2)
+
+                test_fixup('\ue000')
+                test_fixup('\uff61')
+
+        # Surrogates on both sides, no fixup required
+        self.assertTrue('\ud800\udc02' < '\ud84d\udc56')
+
+    def test_islower(self):
+        super().test_islower()
+        self.checkequalnofix(False, '\u1FFc', 'islower')
+        self.assertFalse('\u2167'.islower())
+        self.assertTrue('\u2177'.islower())
+        # non-BMP, uppercase
+        self.assertFalse('\U00010401'.islower())
+        self.assertFalse('\U00010427'.islower())
+        # non-BMP, lowercase
+        self.assertTrue('\U00010429'.islower())
+        self.assertTrue('\U0001044E'.islower())
+        # non-BMP, non-cased
+        self.assertFalse('\U0001F40D'.islower())
+        self.assertFalse('\U0001F46F'.islower())
+
+    def test_isupper(self):
+        super().test_isupper()
+        if not sys.platform.startswith('java'):
+            self.checkequalnofix(False, '\u1FFc', 'isupper')
+        self.assertTrue('\u2167'.isupper())
+        self.assertFalse('\u2177'.isupper())
+        # non-BMP, uppercase
+        self.assertTrue('\U00010401'.isupper())
+        self.assertTrue('\U00010427'.isupper())
+        # non-BMP, lowercase
+        self.assertFalse('\U00010429'.isupper())
+        self.assertFalse('\U0001044E'.isupper())
+        # non-BMP, non-cased
+        self.assertFalse('\U0001F40D'.isupper())
+        self.assertFalse('\U0001F46F'.isupper())
+
+    def test_istitle(self):
+        super().test_istitle()
+        self.checkequalnofix(True, '\u1FFc', 'istitle')
+        self.checkequalnofix(True, 'Greek \u1FFcitlecases ...', 'istitle')
+
+        # non-BMP, uppercase + lowercase
+        self.assertTrue('\U00010401\U00010429'.istitle())
+        self.assertTrue('\U00010427\U0001044E'.istitle())
+        # apparently there are no titlecased (Lt) non-BMP chars in Unicode 6
+        for ch in ['\U00010429', '\U0001044E', '\U0001F40D', '\U0001F46F']:
+            self.assertFalse(ch.istitle(), '{!a} is not title'.format(ch))
+
+    def test_isspace(self):
+        super().test_isspace()
+        self.checkequalnofix(True, '\u2000', 'isspace')
+        self.checkequalnofix(True, '\u200a', 'isspace')
+        self.checkequalnofix(False, '\u2014', 'isspace')
+        # apparently there are no non-BMP spaces chars in Unicode 6
+        for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E',
+                   '\U0001F40D', '\U0001F46F']:
+            self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch))
+
+    def test_isalnum(self):
+        super().test_isalnum()
+        for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E',
+                   '\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']:
+            self.assertTrue(ch.isalnum(), '{!a} is alnum.'.format(ch))
+
+    def test_isalpha(self):
+        super().test_isalpha()
+        self.checkequalnofix(True, '\u1FFc', 'isalpha')
+        # non-BMP, cased
+        self.assertTrue('\U00010401'.isalpha())
+        self.assertTrue('\U00010427'.isalpha())
+        self.assertTrue('\U00010429'.isalpha())
+        self.assertTrue('\U0001044E'.isalpha())
+        # non-BMP, non-cased
+        self.assertFalse('\U0001F40D'.isalpha())
+        self.assertFalse('\U0001F46F'.isalpha())
+
+    @unittest.skipIf(sys.version_info < (3, 7), 'Python lacks str.isascii()')
+    def test_isascii(self):
+        super().test_isascii()
+        self.assertFalse("\u20ac".isascii())
+        self.assertFalse("\U0010ffff".isascii())
+
+    def test_isdecimal(self):
+        self.checkequalnofix(False, '', 'isdecimal')
+        self.checkequalnofix(False, 'a', 'isdecimal')
+        self.checkequalnofix(True, '0', 'isdecimal')
+        self.checkequalnofix(False, '\u2460', 'isdecimal') # CIRCLED DIGIT ONE
+        self.checkequalnofix(False, '\xbc', 'isdecimal') # VULGAR FRACTION ONE QUARTER
+        self.checkequalnofix(True, '\u0660', 'isdecimal') # ARABIC-INDIC DIGIT ZERO
+        self.checkequalnofix(True, '0123456789', 'isdecimal')
+        self.checkequalnofix(False, '0123456789a', 'isdecimal')
+
+        self.checkraises(TypeError, 'abc', 'isdecimal', 42)
+
+        for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E',
+                   '\U0001F40D', '\U0001F46F', '\U00011065', '\U0001F107']:
+            self.assertFalse(ch.isdecimal(), '{!a} is not decimal.'.format(ch))
+        for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']:
+            self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch))
+
+    def test_isdigit(self):
+        super().test_isdigit()
+        self.checkequalnofix(True, '\u2460', 'isdigit')
+        self.checkequalnofix(False, '\xbc', 'isdigit')
+        self.checkequalnofix(True, '\u0660', 'isdigit')
+
+        for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E',
+                   '\U0001F40D', '\U0001F46F', '\U00011065']:
+            self.assertFalse(ch.isdigit(), '{!a} is not a digit.'.format(ch))
+        for ch in ['\U0001D7F6', '\U00011066', '\U000104A0', '\U0001F107']:
+            self.assertTrue(ch.isdigit(), '{!a} is a digit.'.format(ch))
+
+    def test_isnumeric(self):
+        self.checkequalnofix(False, '', 'isnumeric')
+        self.checkequalnofix(False, 'a', 'isnumeric')
+        self.checkequalnofix(True, '0', 'isnumeric')
+        self.checkequalnofix(True, '\u2460', 'isnumeric')
+        self.checkequalnofix(True, '\xbc', 'isnumeric')
+        self.checkequalnofix(True, '\u0660', 'isnumeric')
+        self.checkequalnofix(True, '0123456789', 'isnumeric')
+        self.checkequalnofix(False, '0123456789a', 'isnumeric')
+
+        self.assertRaises(TypeError, "abc".isnumeric, 42)
+
+        for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E',
+                   '\U0001F40D', '\U0001F46F']:
+            self.assertFalse(ch.isnumeric(), '{!a} is not numeric.'.format(ch))
+        for ch in ['\U00011065', '\U0001D7F6', '\U00011066',
+                   '\U000104A0', '\U0001F107']:
+            self.assertTrue(ch.isnumeric(), '{!a} is numeric.'.format(ch))
+
+    def test_isidentifier(self):
+        self.assertTrue("a".isidentifier())
+        self.assertTrue("Z".isidentifier())
+        self.assertTrue("_".isidentifier())
+        self.assertTrue("b0".isidentifier())
+        self.assertTrue("bc".isidentifier())
+        self.assertTrue("b_".isidentifier())
+        self.assertTrue("µ".isidentifier())
+        self.assertTrue("𝔘𝔫𝔦𝔠𝔬𝔡𝔢".isidentifier())
+
+        self.assertFalse(" ".isidentifier())
+        self.assertFalse("[".isidentifier())
+        self.assertFalse("©".isidentifier())
+        self.assertFalse("0".isidentifier())
+
+    def test_isprintable(self):
+        self.assertTrue("".isprintable())
+        self.assertTrue(" ".isprintable())
+        self.assertTrue("abcdefg".isprintable())
+        self.assertFalse("abcdefg\n".isprintable())
+        # some defined Unicode character
+        self.assertTrue("\u0374".isprintable())
+        # undefined character
+        self.assertFalse("\u0378".isprintable())
+        # single surrogate character
+        self.assertFalse("\ud800".isprintable())
+
+        self.assertTrue('\U0001F46F'.isprintable())
+        self.assertFalse('\U000E0020'.isprintable())
+
+    def test_surrogates(self):
+        for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800',
+                  'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'):
+            self.assertTrue(s.islower())
+            self.assertFalse(s.isupper())
+            self.assertFalse(s.istitle())
+        for s in ('A\uD800B\uDFFF', 'A\uDFFFB\uD800',
+                  'A\uD800B\uDFFFA', 'A\uDFFFB\uD800A'):
+            self.assertFalse(s.islower())
+            self.assertTrue(s.isupper())
+            self.assertTrue(s.istitle())
+
+        for meth_name in ('islower', 'isupper', 'istitle'):
+            meth = getattr(str, meth_name)
+            for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF'):
+                self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name))
+
+        for meth_name in ('isalpha', 'isalnum', 'isdigit', 'isspace',
+                          'isdecimal', 'isnumeric',
+                          'isidentifier', 'isprintable'):
+            meth = getattr(str, meth_name)
+            for s in ('\uD800', '\uDFFF', '\uD800\uD800', '\uDFFF\uDFFF',
+                      'a\uD800b\uDFFF', 'a\uDFFFb\uD800',
+                      'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'):
+                self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name))
+
+
+    def test_lower(self):
+        CommonTest.test_lower(self)
+        self.assertEqual('\U00010427'.lower(), '\U0001044F')
+        self.assertEqual('\U00010427\U00010427'.lower(),
+                         '\U0001044F\U0001044F')
+        self.assertEqual('\U00010427\U0001044F'.lower(),
+                         '\U0001044F\U0001044F')
+        self.assertEqual('X\U00010427x\U0001044F'.lower(),
+                         'x\U0001044Fx\U0001044F')
+        self.assertEqual('fi'.lower(), 'fi')
+        self.assertEqual('\u0130'.lower(), '\u0069\u0307')
+        # Special case for GREEK CAPITAL LETTER SIGMA U+03A3
+        self.assertEqual('\u03a3'.lower(), '\u03c3')
+        self.assertEqual('\u0345\u03a3'.lower(), '\u0345\u03c3')
+        self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2')
+        self.assertEqual('A\u0345\u03a3a'.lower(), 'a\u0345\u03c3a')
+        self.assertEqual('A\u0345\u03a3'.lower(), 'a\u0345\u03c2')
+        self.assertEqual('A\u03a3\u0345'.lower(), 'a\u03c2\u0345')
+        self.assertEqual('\u03a3\u0345 '.lower(), '\u03c3\u0345 ')
+        self.assertEqual('\U0008fffe'.lower(), '\U0008fffe')
+        self.assertEqual('\u2177'.lower(), '\u2177')
+
+    def test_casefold(self):
+        self.assertEqual('hello'.casefold(), 'hello')
+        self.assertEqual('hELlo'.casefold(), 'hello')
+        self.assertEqual('ß'.casefold(), 'ss')
+        self.assertEqual('fi'.casefold(), 'fi')
+        self.assertEqual('\u03a3'.casefold(), '\u03c3')
+        self.assertEqual('A\u0345\u03a3'.casefold(), 'a\u03b9\u03c3')
+        self.assertEqual('\u00b5'.casefold(), '\u03bc')
+
+    def test_upper(self):
+        CommonTest.test_upper(self)
+        self.assertEqual('\U0001044F'.upper(), '\U00010427')
+        self.assertEqual('\U0001044F\U0001044F'.upper(),
+                         '\U00010427\U00010427')
+        self.assertEqual('\U00010427\U0001044F'.upper(),
+                         '\U00010427\U00010427')
+        self.assertEqual('X\U00010427x\U0001044F'.upper(),
+                         'X\U00010427X\U00010427')
+        self.assertEqual('fi'.upper(), 'FI')
+        self.assertEqual('\u0130'.upper(), '\u0130')
+        self.assertEqual('\u03a3'.upper(), '\u03a3')
+        self.assertEqual('ß'.upper(), 'SS')
+        self.assertEqual('\u1fd2'.upper(), '\u0399\u0308\u0300')
+        self.assertEqual('\U0008fffe'.upper(), '\U0008fffe')
+        self.assertEqual('\u2177'.upper(), '\u2167')
+
+    def test_capitalize(self):
+        CommonTest.test_capitalize(self)
+        self.assertEqual('\U0001044F'.capitalize(), '\U00010427')
+        self.assertEqual('\U0001044F\U0001044F'.capitalize(),
+                         '\U00010427\U0001044F')
+        self.assertEqual('\U00010427\U0001044F'.capitalize(),
+                         '\U00010427\U0001044F')
+        self.assertEqual('\U0001044F\U00010427'.capitalize(),
+                         '\U00010427\U0001044F')
+        self.assertEqual('X\U00010427x\U0001044F'.capitalize(),
+                         'X\U0001044Fx\U0001044F')
+        self.assertEqual('h\u0130'.capitalize(), 'H\u0069\u0307')
+        exp = '\u0399\u0308\u0300\u0069\u0307'
+        self.assertEqual('\u1fd2\u0130'.capitalize(), exp)
+        if sys.version_info < (3, 8):
+            self.assertEqual('finnish'.capitalize(), 'FInnish')
+        else:
+            self.assertEqual('finnish'.capitalize(), 'Finnish')
+        self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2')
+
+    def test_title(self):
+        super().test_title()
+        self.assertEqual('\U0001044F'.title(), '\U00010427')
+        self.assertEqual('\U0001044F\U0001044F'.title(),
+                         '\U00010427\U0001044F')
+        self.assertEqual('\U0001044F\U0001044F \U0001044F\U0001044F'.title(),
+                         '\U00010427\U0001044F \U00010427\U0001044F')
+        self.assertEqual('\U00010427\U0001044F \U00010427\U0001044F'.title(),
+                         '\U00010427\U0001044F \U00010427\U0001044F')
+        self.assertEqual('\U0001044F\U00010427 \U0001044F\U00010427'.title(),
+                         '\U00010427\U0001044F \U00010427\U0001044F')
+        self.assertEqual('X\U00010427x\U0001044F X\U00010427x\U0001044F'.title(),
+                         'X\U0001044Fx\U0001044F X\U0001044Fx\U0001044F')
+        self.assertEqual('fiNNISH'.title(), 'Finnish')
+        self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy')
+        self.assertEqual('A\u03a3A'.title(), 'A\u03c3a')
+
+    def test_swapcase(self):
+        CommonTest.test_swapcase(self)
+        self.assertEqual('\U0001044F'.swapcase(), '\U00010427')
+        self.assertEqual('\U00010427'.swapcase(), '\U0001044F')
+        self.assertEqual('\U0001044F\U0001044F'.swapcase(),
+                         '\U00010427\U00010427')
+        self.assertEqual('\U00010427\U0001044F'.swapcase(),
+                         '\U0001044F\U00010427')
+        self.assertEqual('\U0001044F\U00010427'.swapcase(),
+                         '\U00010427\U0001044F')
+        self.assertEqual('X\U00010427x\U0001044F'.swapcase(),
+                         'x\U0001044FX\U00010427')
+        self.assertEqual('fi'.swapcase(), 'FI')
+        self.assertEqual('\u0130'.swapcase(), '\u0069\u0307')
+        # Special case for GREEK CAPITAL LETTER SIGMA U+03A3
+        self.assertEqual('\u03a3'.swapcase(), '\u03c3')
+        self.assertEqual('\u0345\u03a3'.swapcase(), '\u0399\u03c3')
+        self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2')
+        self.assertEqual('A\u0345\u03a3a'.swapcase(), 'a\u0399\u03c3A')
+        self.assertEqual('A\u0345\u03a3'.swapcase(), 'a\u0399\u03c2')
+        self.assertEqual('A\u03a3\u0345'.swapcase(), 'a\u03c2\u0399')
+        self.assertEqual('\u03a3\u0345 '.swapcase(), '\u03c3\u0399 ')
+        self.assertEqual('\u03a3'.swapcase(), '\u03c3')
+        self.assertEqual('ß'.swapcase(), 'SS')
+        self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300')
+
+    def test_center(self):
+        CommonTest.test_center(self)
+        self.assertEqual('x'.center(2, '\U0010FFFF'),
+                         'x\U0010FFFF')
+        self.assertEqual('x'.center(3, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF')
+        self.assertEqual('x'.center(4, '\U0010FFFF'),
+                         '\U0010FFFFx\U0010FFFF\U0010FFFF')
+
+    @unittest.skipUnless(sys.maxsize == 2**31 - 1, "requires 32-bit system")
+    @support.cpython_only
+    def test_case_operation_overflow(self):
+        # Issue #22643
+        size = 2**32//12 + 1
+        try:
+            s = "ü" * size
+        except MemoryError:
+            self.skipTest('no enough memory (%.0f MiB required)' % (size / 2**20))
+        try:
+            self.assertRaises(OverflowError, s.upper)
+        finally:
+            del s
+
+    def test_contains(self):
+        # Testing Unicode contains method
+        self.assertIn('a', 'abdb')
+        self.assertIn('a', 'bdab')
+        self.assertIn('a', 'bdaba')
+        self.assertIn('a', 'bdba')
+        self.assertNotIn('a', 'bdb')
+        self.assertIn('a', 'bdba')
+        self.assertIn('a', ('a',1,None))
+        self.assertIn('a', (1,None,'a'))
+        self.assertIn('a', ('a',1,None))
+        self.assertIn('a', (1,None,'a'))
+        self.assertNotIn('a', ('x',1,'y'))
+        self.assertNotIn('a', ('x',1,None))
+        self.assertNotIn('abcd', 'abcxxxx')
+        self.assertIn('ab', 'abcd')
+        self.assertIn('ab', 'abc')
+        self.assertIn('ab', (1,None,'ab'))
+        self.assertIn('', 'abc')
+        self.assertIn('', '')
+        self.assertIn('', 'abc')
+        self.assertNotIn('\0', 'abc')
+        self.assertIn('\0', '\0abc')
+        self.assertIn('\0', 'abc\0')
+        self.assertIn('a', '\0abc')
+        self.assertIn('asdf', 'asdf')
+        self.assertNotIn('asdf', 'asd')
+        self.assertNotIn('asdf', '')
+
+        self.assertRaises(TypeError, "abc".__contains__)
+        # test mixed kinds
+        for fill in ('a', '\u0100', '\U00010300'):
+            fill *= 9
+            for delim in ('c', '\u0102', '\U00010302'):
+                self.assertNotIn(delim, fill)
+                self.assertIn(delim, fill + delim)
+                self.assertNotIn(delim * 2, fill)
+                self.assertIn(delim * 2, fill + delim * 2)
+
+    def test_issue18183(self):
+        '\U00010000\U00100000'.lower()
+        '\U00010000\U00100000'.casefold()
+        '\U00010000\U00100000'.upper()
+        '\U00010000\U00100000'.capitalize()
+        '\U00010000\U00100000'.title()
+        '\U00010000\U00100000'.swapcase()
+        '\U00100000'.center(3, '\U00010000')
+        '\U00100000'.ljust(3, '\U00010000')
+        '\U00100000'.rjust(3, '\U00010000')
+
+    def test_format(self):
+        self.assertEqual(''.format(), '')
+        self.assertEqual('a'.format(), 'a')
+        self.assertEqual('ab'.format(), 'ab')
+        self.assertEqual('a{{'.format(), 'a{')
+        self.assertEqual('a}}'.format(), 'a}')
+        self.assertEqual('{{b'.format(), '{b')
+        self.assertEqual('}}b'.format(), '}b')
+        self.assertEqual('a{{b'.format(), 'a{b')
+
+        # examples from the PEP:
+        import datetime
+        self.assertEqual("My name is {0}".format('Fred'), "My name is Fred")
+        self.assertEqual("My name is {0[name]}".format(dict(name='Fred')),
+                         "My name is Fred")
+        self.assertEqual("My name is {0} :-{{}}".format('Fred'),
+                         "My name is Fred :-{}")
+
+        d = datetime.date(2007, 8, 18)
+        self.assertEqual("The year is {0.year}".format(d),
+                         "The year is 2007")
+
+        # classes we'll use for testing
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        class D:
+            def __init__(self, x):
+                self.x = x
+            def __format__(self, spec):
+                return str(self.x)
+
+        # class with __str__, but no __format__
+        class E:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return 'E(' + self.x + ')'
+
+        # class with __repr__, but no __format__ or __str__
+        class F:
+            def __init__(self, x):
+                self.x = x
+            def __repr__(self):
+                return 'F(' + self.x + ')'
+
+        # class with __format__ that forwards to string, for some format_spec's
+        class G:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return "string is " + self.x
+            def __format__(self, format_spec):
+                if format_spec == 'd':
+                    return 'G(' + self.x + ')'
+                return object.__format__(self, format_spec)
+
+        class I(datetime.date):
+            def __format__(self, format_spec):
+                return self.strftime(format_spec)
+
+        class J(int):
+            def __format__(self, format_spec):
+                return int.__format__(self * 2, format_spec)
+
+        class M:
+            def __init__(self, x):
+                self.x = x
+            def __repr__(self):
+                return 'M(' + self.x + ')'
+            __str__ = None
+
+        class N:
+            def __init__(self, x):
+                self.x = x
+            def __repr__(self):
+                return 'N(' + self.x + ')'
+            __format__ = None
+
+        self.assertEqual(''.format(), '')
+        self.assertEqual('abc'.format(), 'abc')
+        self.assertEqual('{0}'.format('abc'), 'abc')
+        self.assertEqual('{0:}'.format('abc'), 'abc')
+#        self.assertEqual('{ 0 }'.format('abc'), 'abc')
+        self.assertEqual('X{0}'.format('abc'), 'Xabc')
+        self.assertEqual('{0}X'.format('abc'), 'abcX')
+        self.assertEqual('X{0}Y'.format('abc'), 'XabcY')
+        self.assertEqual('{1}'.format(1, 'abc'), 'abc')
+        self.assertEqual('X{1}'.format(1, 'abc'), 'Xabc')
+        self.assertEqual('{1}X'.format(1, 'abc'), 'abcX')
+        self.assertEqual('X{1}Y'.format(1, 'abc'), 'XabcY')
+        self.assertEqual('{0}'.format(-15), '-15')
+        self.assertEqual('{0}{1}'.format(-15, 'abc'), '-15abc')
+        self.assertEqual('{0}X{1}'.format(-15, 'abc'), '-15Xabc')
+        self.assertEqual('{{'.format(), '{')
+        self.assertEqual('}}'.format(), '}')
+        self.assertEqual('{{}}'.format(), '{}')
+        self.assertEqual('{{x}}'.format(), '{x}')
+        self.assertEqual('{{{0}}}'.format(123), '{123}')
+        self.assertEqual('{{{{0}}}}'.format(), '{{0}}')
+        self.assertEqual('}}{{'.format(), '}{')
+        self.assertEqual('}}x{{'.format(), '}x{')
+
+        # weird field names
+        self.assertEqual("{0[foo-bar]}".format({'foo-bar':'baz'}), 'baz')
+        self.assertEqual("{0[foo bar]}".format({'foo bar':'baz'}), 'baz')
+        self.assertEqual("{0[ ]}".format({' ':3}), '3')
+
+        self.assertEqual('{foo._x}'.format(foo=C(20)), '20')
+        self.assertEqual('{1}{0}'.format(D(10), D(20)), '2010')
+        self.assertEqual('{0._x.x}'.format(C(D('abc'))), 'abc')
+        self.assertEqual('{0[0]}'.format(['abc', 'def']), 'abc')
+        self.assertEqual('{0[1]}'.format(['abc', 'def']), 'def')
+        self.assertEqual('{0[1][0]}'.format(['abc', ['def']]), 'def')
+        self.assertEqual('{0[1][0].x}'.format(['abc', [D('def')]]), 'def')
+
+        # strings
+        self.assertEqual('{0:.3s}'.format('abc'), 'abc')
+        self.assertEqual('{0:.3s}'.format('ab'), 'ab')
+        self.assertEqual('{0:.3s}'.format('abcdef'), 'abc')
+        self.assertEqual('{0:.0s}'.format('abcdef'), '')
+        self.assertEqual('{0:3.3s}'.format('abc'), 'abc')
+        self.assertEqual('{0:2.3s}'.format('abc'), 'abc')
+        self.assertEqual('{0:2.2s}'.format('abc'), 'ab')
+        self.assertEqual('{0:3.2s}'.format('abc'), 'ab ')
+        self.assertEqual('{0:x<0s}'.format('result'), 'result')
+        self.assertEqual('{0:x<5s}'.format('result'), 'result')
+        self.assertEqual('{0:x<6s}'.format('result'), 'result')
+        self.assertEqual('{0:x<7s}'.format('result'), 'resultx')
+        self.assertEqual('{0:x<8s}'.format('result'), 'resultxx')
+        self.assertEqual('{0: <7s}'.format('result'), 'result ')
+        self.assertEqual('{0:<7s}'.format('result'), 'result ')
+        self.assertEqual('{0:>7s}'.format('result'), ' result')
+        self.assertEqual('{0:>8s}'.format('result'), '  result')
+        self.assertEqual('{0:^8s}'.format('result'), ' result ')
+        self.assertEqual('{0:^9s}'.format('result'), ' result  ')
+        self.assertEqual('{0:^10s}'.format('result'), '  result  ')
+        self.assertEqual('{0:10000}'.format('a'), 'a' + ' ' * 9999)
+        self.assertEqual('{0:10000}'.format(''), ' ' * 10000)
+        self.assertEqual('{0:10000000}'.format(''), ' ' * 10000000)
+
+        # issue 12546: use \x00 as a fill character
+        self.assertEqual('{0:\x00<6s}'.format('foo'), 'foo\x00\x00\x00')
+        self.assertEqual('{0:\x01<6s}'.format('foo'), 'foo\x01\x01\x01')
+        self.assertEqual('{0:\x00^6s}'.format('foo'), '\x00foo\x00\x00')
+        self.assertEqual('{0:^6s}'.format('foo'), ' foo  ')
+
+        self.assertEqual('{0:\x00<6}'.format(3), '3\x00\x00\x00\x00\x00')
+        self.assertEqual('{0:\x01<6}'.format(3), '3\x01\x01\x01\x01\x01')
+        self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00')
+        self.assertEqual('{0:<6}'.format(3), '3     ')
+
+        self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00')
+        self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01')
+        self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00')
+        self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ')
+
+        self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00')
+        self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01')
+        self.assertEqual('{0:\x00^12}'.format(3+2.0j), '\x00\x00\x00(3+2j)\x00\x00\x00')
+        self.assertEqual('{0:^12}'.format(3+2.0j), '   (3+2j)   ')
+
+        # format specifiers for user defined type
+        self.assertEqual('{0:abc}'.format(C()), 'abc')
+
+        # !r, !s and !a coercions
+        self.assertEqual('{0!s}'.format('Hello'), 'Hello')
+        self.assertEqual('{0!s:}'.format('Hello'), 'Hello')
+        self.assertEqual('{0!s:15}'.format('Hello'), 'Hello          ')
+        self.assertEqual('{0!s:15s}'.format('Hello'), 'Hello          ')
+        self.assertEqual('{0!r}'.format('Hello'), "'Hello'")
+        self.assertEqual('{0!r:}'.format('Hello'), "'Hello'")
+        self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)')
+        self.assertEqual('{0!r}'.format('\u0378'), "'\\u0378'") # nonprintable
+        self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'")  # printable
+        self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)')
+        self.assertEqual('{0!a}'.format('Hello'), "'Hello'")
+        self.assertEqual('{0!a}'.format('\u0378'), "'\\u0378'") # nonprintable
+        self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable
+        self.assertEqual('{0!a:}'.format('Hello'), "'Hello'")
+        self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)')
+        self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)')
+
+        # test fallback to object.__format__
+        self.assertEqual('{0}'.format({}), '{}')
+        self.assertEqual('{0}'.format([]), '[]')
+        self.assertEqual('{0}'.format([1]), '[1]')
+
+        self.assertEqual('{0:d}'.format(G('data')), 'G(data)')
+        self.assertEqual('{0!s}'.format(G('data')), 'string is data')
+
+        self.assertRaises(TypeError, '{0:^10}'.format, E('data'))
+        self.assertRaises(TypeError, '{0:^10s}'.format, E('data'))
+        self.assertRaises(TypeError, '{0:>15s}'.format, G('data'))
+
+        self.assertEqual("{0:date: %Y-%m-%d}".format(I(year=2007,
+                                                       month=8,
+                                                       day=27)),
+                         "date: 2007-08-27")
+
+        # test deriving from a builtin type and overriding __format__
+        self.assertEqual("{0}".format(J(10)), "20")
+
+
+        # string format specifiers
+        self.assertEqual('{0:}'.format('a'), 'a')
+
+        # computed format specifiers
+        self.assertEqual("{0:.{1}}".format('hello world', 5), 'hello')
+        self.assertEqual("{0:.{1}s}".format('hello world', 5), 'hello')
+        self.assertEqual("{0:.{precision}s}".format('hello world', precision=5), 'hello')
+        self.assertEqual("{0:{width}.{precision}s}".format('hello world', width=10, precision=5), 'hello     ')
+        self.assertEqual("{0:{width}.{precision}s}".format('hello world', width='10', precision='5'), 'hello     ')
+
+        # test various errors
+        self.assertRaises(ValueError, '{'.format)
+        self.assertRaises(ValueError, '}'.format)
+        self.assertRaises(ValueError, 'a{'.format)
+        self.assertRaises(ValueError, 'a}'.format)
+        self.assertRaises(ValueError, '{a'.format)
+        self.assertRaises(ValueError, '}a'.format)
+        self.assertRaises(IndexError, '{0}'.format)
+        self.assertRaises(IndexError, '{1}'.format, 'abc')
+        self.assertRaises(KeyError,   '{x}'.format)
+        self.assertRaises(ValueError, "}{".format)
+        self.assertRaises(ValueError, "abc{0:{}".format)
+        self.assertRaises(ValueError, "{0".format)
+        self.assertRaises(IndexError, "{0.}".format)
+        self.assertRaises(ValueError, "{0.}".format, 0)
+        self.assertRaises(ValueError, "{0[}".format)
+        self.assertRaises(ValueError, "{0[}".format, [])
+        self.assertRaises(KeyError,   "{0]}".format)
+        self.assertRaises(ValueError, "{0.[]}".format, 0)
+        self.assertRaises(ValueError, "{0..foo}".format, 0)
+        self.assertRaises(ValueError, "{0[0}".format, 0)
+        self.assertRaises(ValueError, "{0[0:foo}".format, 0)
+        self.assertRaises(KeyError,   "{c]}".format)
+        self.assertRaises(ValueError, "{{ {{{0}}".format, 0)
+        self.assertRaises(ValueError, "{0}}".format, 0)
+        self.assertRaises(KeyError,   "{foo}".format, bar=3)
+        self.assertRaises(ValueError, "{0!x}".format, 3)
+        self.assertRaises(ValueError, "{0!}".format, 0)
+        self.assertRaises(ValueError, "{0!rs}".format, 0)
+        self.assertRaises(ValueError, "{!}".format)
+        self.assertRaises(IndexError, "{:}".format)
+        self.assertRaises(IndexError, "{:s}".format)
+        self.assertRaises(IndexError, "{}".format)
+        big = "23098475029384702983476098230754973209482573"
+        self.assertRaises(ValueError, ("{" + big + "}").format)
+        self.assertRaises(ValueError, ("{[" + big + "]}").format, [0])
+
+        # issue 6089
+        self.assertRaises(ValueError, "{0[0]x}".format, [None])
+        self.assertRaises(ValueError, "{0[0](10)}".format, [None])
+
+        # can't have a replacement on the field name portion
+        self.assertRaises(TypeError, '{0[{1}]}'.format, 'abcdefg', 4)
+
+        # exceed maximum recursion depth
+        self.assertRaises(ValueError, "{0:{1:{2}}}".format, 'abc', 's', '')
+        self.assertRaises(ValueError, "{0:{1:{2:{3:{4:{5:{6}}}}}}}".format,
+                          0, 1, 2, 3, 4, 5, 6, 7)
+
+        # string format spec errors
+        self.assertRaises(ValueError, "{0:-s}".format, '')
+        self.assertRaises(ValueError, format, "", "-")
+        self.assertRaises(ValueError, "{0:=s}".format, '')
+
+        # Alternate formatting is not supported
+        self.assertRaises(ValueError, format, '', '#')
+        self.assertRaises(ValueError, format, '', '#20')
+
+        # Non-ASCII
+        self.assertEqual("{0:s}{1:s}".format("ABC", "\u0410\u0411\u0412"),
+                         'ABC\u0410\u0411\u0412')
+        self.assertEqual("{0:.3s}".format("ABC\u0410\u0411\u0412"),
+                         'ABC')
+        self.assertEqual("{0:.0s}".format("ABC\u0410\u0411\u0412"),
+                         '')
+
+        self.assertEqual("{[{}]}".format({"{}": 5}), "5")
+        self.assertEqual("{[{}]}".format({"{}" : "a"}), "a")
+        self.assertEqual("{[{]}".format({"{" : "a"}), "a")
+        self.assertEqual("{[}]}".format({"}" : "a"}), "a")
+        self.assertEqual("{[[]}".format({"[" : "a"}), "a")
+        self.assertEqual("{[!]}".format({"!" : "a"}), "a")
+        self.assertRaises(ValueError, "{a{}b}".format, 42)
+        self.assertRaises(ValueError, "{a{b}".format, 42)
+        self.assertRaises(ValueError, "{[}".format, 42)
+
+        self.assertEqual("0x{:0{:d}X}".format(0x0,16), "0x0000000000000000")
+
+        # Blocking fallback
+        m = M('data')
+        self.assertEqual("{!r}".format(m), 'M(data)')
+        self.assertRaises(TypeError, "{!s}".format, m)
+        self.assertRaises(TypeError, "{}".format, m)
+        n = N('data')
+        self.assertEqual("{!r}".format(n), 'N(data)')
+        self.assertEqual("{!s}".format(n), 'N(data)')
+        self.assertRaises(TypeError, "{}".format, n)
+
+    @unittest.skipIf(sys.version_info < (3, 6), 'Python str.format_map() test requires Py3.6+')
+    def test_format_map(self):
+        self.assertEqual(''.format_map({}), '')
+        self.assertEqual('a'.format_map({}), 'a')
+        self.assertEqual('ab'.format_map({}), 'ab')
+        self.assertEqual('a{{'.format_map({}), 'a{')
+        self.assertEqual('a}}'.format_map({}), 'a}')
+        self.assertEqual('{{b'.format_map({}), '{b')
+        self.assertEqual('}}b'.format_map({}), '}b')
+        self.assertEqual('a{{b'.format_map({}), 'a{b')
+
+        # using mappings
+        class Mapping(dict):
+            def __missing__(self, key):
+                return key
+        self.assertEqual('{hello}'.format_map(Mapping()), 'hello')
+        self.assertEqual('{a} {world}'.format_map(Mapping(a='hello')), 'hello world')
+
+        class InternalMapping:
+            def __init__(self):
+                self.mapping = {'a': 'hello'}
+            def __getitem__(self, key):
+                return self.mapping[key]
+        self.assertEqual('{a}'.format_map(InternalMapping()), 'hello')
+
+
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+        self.assertEqual('{foo._x}'.format_map({'foo': C(20)}), '20')
+
+        # test various errors
+        self.assertRaises(TypeError, ''.format_map)
+        self.assertRaises(TypeError, 'a'.format_map)
+
+        self.assertRaises(ValueError, '{'.format_map, {})
+        self.assertRaises(ValueError, '}'.format_map, {})
+        self.assertRaises(ValueError, 'a{'.format_map, {})
+        self.assertRaises(ValueError, 'a}'.format_map, {})
+        self.assertRaises(ValueError, '{a'.format_map, {})
+        self.assertRaises(ValueError, '}a'.format_map, {})
+
+        # issue #12579: can't supply positional params to format_map
+        self.assertRaises(ValueError, '{}'.format_map, {'a' : 2})
+        self.assertRaises(ValueError, '{}'.format_map, 'a')
+        self.assertRaises(ValueError, '{a} {}'.format_map, {"a" : 2, "b" : 1})
+
+        ZERO = 0
+        class BadMapping:
+            def __getitem__(self, key):
+                return 1 / ZERO
+        self.assertRaises(KeyError, '{a}'.format_map, {})
+        self.assertRaises(TypeError, '{a}'.format_map, [])
+        self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping())
+
+    def test_format_huge_precision(self):
+        format_string = ".{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format(2.34, format_string)
+
+    def test_format_huge_width(self):
+        format_string = "{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format(2.34, format_string)
+
+    def test_format_huge_item_number(self):
+        format_string = "{{{}:.6f}}".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format_string.format(2.34)
+
+    def test_format_auto_numbering(self):
+        class C:
+            def __init__(self, x=100):
+                self._x = x
+            def __format__(self, spec):
+                return spec
+
+        self.assertEqual('{}'.format(10), '10')
+        self.assertEqual('{:5}'.format('s'), 's    ')
+        self.assertEqual('{!r}'.format('s'), "'s'")
+        self.assertEqual('{._x}'.format(C(10)), '10')
+        self.assertEqual('{[1]}'.format([1, 2]), '2')
+        self.assertEqual('{[a]}'.format({'a':4, 'b':2}), '4')
+        self.assertEqual('a{}b{}c'.format(0, 1), 'a0b1c')
+
+        self.assertEqual('a{:{}}b'.format('x', '^10'), 'a    x     b')
+        self.assertEqual('a{:{}x}b'.format(20, '#'), 'a0x14b')
+
+        # can't mix and match numbering and auto-numbering
+        self.assertRaises(ValueError, '{}{1}'.format, 1, 2)
+        self.assertRaises(ValueError, '{1}{}'.format, 1, 2)
+        self.assertRaises(ValueError, '{:{1}}'.format, 1, 2)
+        self.assertRaises(ValueError, '{0:{}}'.format, 1, 2)
+
+        # can mix and match auto-numbering and named
+        self.assertEqual('{f}{}'.format(4, f='test'), 'test4')
+        self.assertEqual('{}{f}'.format(4, f='test'), '4test')
+        self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3')
+        self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g')
+
+    def test_formatting(self):
+        MixinStrUnicodeUserStringTest.test_formatting(self)
+        # Testing Unicode formatting strings...
+        self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc')
+        self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000,  3.00')
+        self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, -2, 3), 'abc, abc, 1, -2.000000,  3.00')
+        self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.5), 'abc, abc, -1, -2.000000,  3.50')
+        self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 3.57), 'abc, abc, -1, -2.000000,  3.57')
+        self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", -1, -2, 1003.57), 'abc, abc, -1, -2.000000, 1003.57')
+        if not sys.platform.startswith('java'):
+            self.assertEqual("%r, %r" % (b"abc", "abc"), "b'abc', 'abc'")
+            self.assertEqual("%r" % ("\u1234",), "'\u1234'")
+            self.assertEqual("%a" % ("\u1234",), "'\\u1234'")
+        self.assertEqual("%(x)s, %(y)s" % {'x':"abc", 'y':"def"}, 'abc, def')
+        self.assertEqual("%(x)s, %(\xfc)s" % {'x':"abc", '\xfc':"def"}, 'abc, def')
+
+        self.assertEqual('%c' % 0x1234, '\u1234')
+        self.assertEqual('%c' % 0x21483, '\U00021483')
+        self.assertRaises(OverflowError, "%c".__mod__, (0x110000,))
+        self.assertEqual('%c' % '\U00021483', '\U00021483')
+        self.assertRaises(TypeError, "%c".__mod__, "aa")
+        self.assertRaises(ValueError, "%.1\u1032f".__mod__, (1.0/3))
+        self.assertRaises(TypeError, "%i".__mod__, "aa")
+
+        # formatting jobs delegated from the string implementation:
+        self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
+        self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
+        self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
+        self.assertEqual('...%(foo)s...' % {'foo':"abc"}, '...abc...')
+        self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123},  '...abc...')
+        self.assertEqual('...%(foo)s...' % {'foo':"abc",'def':123}, '...abc...')
+        self.assertEqual('...%s...%s...%s...%s...' % (1,2,3,"abc"), '...1...2...3...abc...')
+        self.assertEqual('...%%...%%s...%s...%s...%s...%s...' % (1,2,3,"abc"), '...%...%s...1...2...3...abc...')
+        self.assertEqual('...%s...' % "abc", '...abc...')
+        self.assertEqual('%*s' % (5,'abc',), '  abc')
+        self.assertEqual('%*s' % (-5,'abc',), 'abc  ')
+        self.assertEqual('%*.*s' % (5,2,'abc',), '   ab')
+        self.assertEqual('%*.*s' % (5,3,'abc',), '  abc')
+        self.assertEqual('%i %*.*s' % (10, 5,3,'abc',), '10   abc')
+        self.assertEqual('%i%s %*.*s' % (10, 3, 5, 3, 'abc',), '103   abc')
+        self.assertEqual('%c' % 'a', 'a')
+        class Wrapper:
+            def __str__(self):
+                return '\u1234'
+        self.assertEqual('%s' % Wrapper(), '\u1234')
+
+        # issue 3382
+        NAN = float('nan')
+        INF = float('inf')
+        self.assertEqual('%f' % NAN, 'nan')
+        self.assertEqual('%F' % NAN, 'NAN')
+        self.assertEqual('%f' % INF, 'inf')
+        self.assertEqual('%F' % INF, 'INF')
+
+        # PEP 393
+        self.assertEqual('%.1s' % "a\xe9\u20ac", 'a')
+        self.assertEqual('%.2s' % "a\xe9\u20ac", 'a\xe9')
+
+        #issue 19995
+        class PseudoInt:
+            def __init__(self, value):
+                self.value = int(value)
+            def __int__(self):
+                return self.value
+            def __index__(self):
+                return self.value
+        class PseudoFloat:
+            def __init__(self, value):
+                self.value = float(value)
+            def __int__(self):
+                return int(self.value)
+        pi = PseudoFloat(3.1415)
+        letter_m = PseudoInt(109)
+        self.assertEqual('%x' % 42, '2a')
+        self.assertEqual('%X' % 15, 'F')
+        self.assertEqual('%o' % 9, '11')
+        self.assertEqual('%c' % 109, 'm')
+        self.assertEqual('%x' % letter_m, '6d')
+        self.assertEqual('%X' % letter_m, '6D')
+        self.assertEqual('%o' % letter_m, '155')
+        self.assertEqual('%c' % letter_m, 'm')
+        if sys.version_info >= (3, 5):
+            self.assertRaisesRegex(TypeError, '%x format: an integer is required, not float', operator.mod, '%x', 3.14),
+            self.assertRaisesRegex(TypeError, '%X format: an integer is required, not float', operator.mod, '%X', 2.11),
+            self.assertRaisesRegex(TypeError, '%o format: an integer is required, not float', operator.mod, '%o', 1.79),
+            self.assertRaisesRegex(TypeError, '%x format: an integer is required, not PseudoFloat', operator.mod, '%x', pi),
+            self.assertRaises(TypeError, operator.mod, '%c', pi),
+
+    def test_formatting_with_enum(self):
+        # issue18780
+        import enum
+        class Float(float, enum.Enum):
+            PI = 3.1415926
+        class Int(enum.IntEnum):
+            IDES = 15
+        class Str(str, enum.Enum):
+            ABC = 'abc'
+        # Testing Unicode formatting strings...
+        self.assertEqual(("%s, %s" % (Str.ABC, Str.ABC)).replace("Str.", ""),
+                         'ABC, ABC')
+        self.assertEqual(("%s, %s, %d, %i, %u, %f, %5.2f" %
+                        (Str.ABC, Str.ABC,
+                         Int.IDES, Int.IDES, Int.IDES,
+                         Float.PI, Float.PI)).replace("Str.", ""),
+                         'ABC, ABC, 15, 15, 15, 3.141593,  3.14')
+
+        # formatting jobs delegated from the string implementation:
+        self.assertEqual(('...%(foo)s...' % {'foo':Str.ABC}).replace("Str.", ""),
+                         '...ABC...')
+        self.assertEqual(('...%(foo)s...' % {'foo':Int.IDES}).replace("Int.", ""),
+                         '...IDES...' if sys.version_info < (3,11) else '...15...')
+        self.assertEqual('...%(foo)i...' % {'foo':Int.IDES},
+                         '...15...')
+        self.assertEqual('...%(foo)d...' % {'foo':Int.IDES},
+                         '...15...')
+        self.assertEqual('...%(foo)u...' % {'foo':Int.IDES, 'def':Float.PI},
+                         '...15...')
+        self.assertEqual('...%(foo)f...' % {'foo':Float.PI,'def':123},
+                         '...3.141593...')
+
+    def test_formatting_huge_precision(self):
+        format_string = "%.{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format_string % 2.34
+
+    @unittest.skip('BROKEN!')
+    def test_issue28598_strsubclass_rhs(self):
+        # A subclass of str with an __rmod__ method should be able to hook
+        # into the % operator
+        class SubclassedStr(str):
+            def __rmod__(self, other):
+                return 'Success, self.__rmod__({!r}) was called'.format(other)
+        self.assertEqual('lhs %% %r' % SubclassedStr('rhs'),
+                         "Success, self.__rmod__('lhs %% %r') was called")
+
+    @support.cpython_only
+    def test_formatting_huge_precision_c_limits(self):
+        from _testcapi import INT_MAX
+        format_string = "%.{}f".format(INT_MAX + 1)
+        with self.assertRaises(ValueError):
+            result = format_string % 2.34
+
+    def test_formatting_huge_width(self):
+        format_string = "%{}f".format(sys.maxsize + 1)
+        with self.assertRaises(ValueError):
+            result = format_string % 2.34
+
+    def test_startswith_endswith_errors(self):
+        for meth in ('foo'.startswith, 'foo'.endswith):
+            with self.assertRaises(TypeError) as cm:
+                meth(['f'])
+            exc = str(cm.exception)
+            self.assertIn('str', exc)
+            self.assertIn('tuple', exc)
+
+    @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
+    def test_format_float(self):
+        # should not format with a comma, but always with C locale
+        self.assertEqual('1.0', '%.1f' % 1.0)
+
+    def test_constructor(self):
+        # unicode(obj) tests (this maps to PyObject_Unicode() at C level)
+
+        self.assertEqual(
+            str('unicode remains unicode'),
+            'unicode remains unicode'
+        )
+
+        for text in ('ascii', '\xe9', '\u20ac', '\U0010FFFF'):
+            subclass = StrSubclass(text)
+            self.assertEqual(str(subclass), text)
+            self.assertEqual(len(subclass), len(text))
+            if text == 'ascii':
+                self.assertEqual(subclass.encode('ascii'), b'ascii')
+                self.assertEqual(subclass.encode('utf-8'), b'ascii')
+
+        self.assertEqual(
+            str('strings are converted to unicode'),
+            'strings are converted to unicode'
+        )
+
+        class StringCompat:
+            def __init__(self, x):
+                self.x = x
+            def __str__(self):
+                return self.x
+
+        self.assertEqual(
+            str(StringCompat('__str__ compatible objects are recognized')),
+            '__str__ compatible objects are recognized'
+        )
+
+        # unicode(obj) is compatible to str():
+
+        o = StringCompat('unicode(obj) is compatible to str()')
+        self.assertEqual(str(o), 'unicode(obj) is compatible to str()')
+        self.assertEqual(str(o), 'unicode(obj) is compatible to str()')
+
+        for obj in (123, 123.45, 123):
+            self.assertEqual(str(obj), str(str(obj)))
+
+        # unicode(obj, encoding, error) tests (this maps to
+        # PyUnicode_FromEncodedObject() at C level)
+
+        if not sys.platform.startswith('java'):
+            self.assertRaises(
+                TypeError,
+                str,
+                'decoding unicode is not supported',
+                'utf-8',
+                'strict'
+            )
+
+        self.assertEqual(
+            str(b'strings are decoded to unicode', 'utf-8', 'strict'),
+            'strings are decoded to unicode'
+        )
+
+        if not sys.platform.startswith('java'):
+            self.assertEqual(
+                str(
+                    memoryview(b'character buffers are decoded to unicode'),
+                    'utf-8',
+                    'strict'
+                ),
+                'character buffers are decoded to unicode'
+            )
+
+        self.assertRaises(TypeError, str, 42, 42, 42)
+
+    def test_constructor_keyword_args(self):
+        """Pass various keyword argument combinations to the constructor."""
+        # The object argument can be passed as a keyword.
+        self.assertEqual(str(object='foo'), 'foo')
+        self.assertEqual(str(object=b'foo', encoding='utf-8'), 'foo')
+        # The errors argument without encoding triggers "decode" mode.
+        self.assertEqual(str(b'foo', errors='strict'), 'foo')  # not "b'foo'"
+        self.assertEqual(str(object=b'foo', errors='strict'), 'foo')
+
+    def test_constructor_defaults(self):
+        """Check the constructor argument defaults."""
+        # The object argument defaults to '' or b''.
+        self.assertEqual(str(), '')
+        self.assertEqual(str(errors='strict'), '')
+        utf8_cent = '¢'.encode('utf-8')
+        # The encoding argument defaults to utf-8.
+        self.assertEqual(str(utf8_cent, errors='strict'), '¢')
+        # The errors argument defaults to strict.
+        self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii')
+
+    @unittest.skipIf(sys.version_info < (3, 6), 'Python utf-7 codec test requires Py3.6+')
+    def test_codecs_utf7(self):
+        utfTests = [
+            ('A\u2262\u0391.', b'A+ImIDkQ.'),             # RFC2152 example
+            ('Hi Mom -\u263a-!', b'Hi Mom -+Jjo--!'),     # RFC2152 example
+            ('\u65E5\u672C\u8A9E', b'+ZeVnLIqe-'),        # RFC2152 example
+            ('Item 3 is \u00a31.', b'Item 3 is +AKM-1.'), # RFC2152 example
+            ('+', b'+-'),
+            ('+-', b'+--'),
+            ('+?', b'+-?'),
+            (r'\?', b'+AFw?'),
+            ('+?', b'+-?'),
+            (r'\\?', b'+AFwAXA?'),
+            (r'\\\?', b'+AFwAXABc?'),
+            (r'++--', b'+-+---'),
+            ('\U000abcde', b'+2m/c3g-'),                  # surrogate pairs
+            ('/', b'/'),
+        ]
+
+        for (x, y) in utfTests:
+            self.assertEqual(x.encode('utf-7'), y)
+
+        # Unpaired surrogates are passed through
+        self.assertEqual('\uD801'.encode('utf-7'), b'+2AE-')
+        self.assertEqual('\uD801x'.encode('utf-7'), b'+2AE-x')
+        self.assertEqual('\uDC01'.encode('utf-7'), b'+3AE-')
+        self.assertEqual('\uDC01x'.encode('utf-7'), b'+3AE-x')
+        self.assertEqual(b'+2AE-'.decode('utf-7'), '\uD801')
+        self.assertEqual(b'+2AE-x'.decode('utf-7'), '\uD801x')
+        self.assertEqual(b'+3AE-'.decode('utf-7'), '\uDC01')
+        self.assertEqual(b'+3AE-x'.decode('utf-7'), '\uDC01x')
+
+        self.assertEqual('\uD801\U000abcde'.encode('utf-7'), b'+2AHab9ze-')
+        self.assertEqual(b'+2AHab9ze-'.decode('utf-7'), '\uD801\U000abcde')
+
+        # Issue #2242: crash on some Windows/MSVC versions
+        self.assertEqual(b'+\xc1'.decode('utf-7', 'ignore'), '')
+
+        # Direct encoded characters
+        set_d = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'(),-./:?"
+        # Optional direct characters
+        set_o = '!"#$%&*;<=>@[]^_`{|}'
+        for c in set_d:
+            self.assertEqual(c.encode('utf7'), c.encode('ascii'))
+            self.assertEqual(c.encode('ascii').decode('utf7'), c)
+        for c in set_o:
+            self.assertEqual(c.encode('ascii').decode('utf7'), c)
+
+        if sys.version_info >= (3, 8):
+            with self.assertRaisesRegex(UnicodeDecodeError,
+                                        'ill-formed sequence'):
+                b'+@'.decode('utf-7')
+
+    def test_codecs_utf8(self):
+        self.assertEqual(''.encode('utf-8'), b'')
+        self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac')
+        self.assertEqual('\U00010002'.encode('utf-8'), b'\xf0\x90\x80\x82')
+        self.assertEqual('\U00023456'.encode('utf-8'), b'\xf0\xa3\x91\x96')
+        self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80')
+        self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80')
+        self.assertEqual(('\U00010002'*10).encode('utf-8'),
+                         b'\xf0\x90\x80\x82'*10)
+        self.assertEqual(
+            '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f'
+            '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00'
+            '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c'
+            '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067'
+            '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das'
+            ' Nunstuck git und'.encode('utf-8'),
+            b'\xe6\xad\xa3\xe7\xa2\xba\xe3\x81\xab\xe8\xa8\x80\xe3\x81'
+            b'\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3\xe3\x81\xaf\xe3'
+            b'\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3\x81\xbe'
+            b'\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83'
+            b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8'
+            b'\xaa\x9e\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81'
+            b'\xe3\x81\x82\xe3\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81'
+            b'\x9f\xe3\x82\x89\xe3\x82\x81\xe3\x81\xa7\xe3\x81\x99\xe3'
+            b'\x80\x82\xe5\xae\x9f\xe9\x9a\x9b\xe3\x81\xab\xe3\x81\xaf'
+            b'\xe3\x80\x8cWenn ist das Nunstuck git und'
+        )
+
+        # UTF-8 specific decoding tests
+        self.assertEqual(str(b'\xf0\xa3\x91\x96', 'utf-8'), '\U00023456' )
+        self.assertEqual(str(b'\xf0\x90\x80\x82', 'utf-8'), '\U00010002' )
+        self.assertEqual(str(b'\xe2\x82\xac', 'utf-8'), '\u20ac' )
+
+        # Other possible utf-8 test cases:
+        # * strict decoding testing for all of the
+        #   UTF8_ERROR cases in PyUnicode_DecodeUTF8
+
+    def test_utf8_decode_valid_sequences(self):
+        sequences = [
+            # single byte
+            (b'\x00', '\x00'), (b'a', 'a'), (b'\x7f', '\x7f'),
+            # 2 bytes
+            (b'\xc2\x80', '\x80'), (b'\xdf\xbf', '\u07ff'),
+            # 3 bytes
+            (b'\xe0\xa0\x80', '\u0800'), (b'\xed\x9f\xbf', '\ud7ff'),
+            (b'\xee\x80\x80', '\uE000'), (b'\xef\xbf\xbf', '\uffff'),
+            # 4 bytes
+            (b'\xF0\x90\x80\x80', '\U00010000'),
+            (b'\xf4\x8f\xbf\xbf', '\U0010FFFF')
+        ]
+        for seq, res in sequences:
+            self.assertEqual(seq.decode('utf-8'), res)
+
+
+    def test_utf8_decode_invalid_sequences(self):
+        # continuation bytes in a sequence of 2, 3, or 4 bytes
+        continuation_bytes = [bytes([x]) for x in range(0x80, 0xC0)]
+        # start bytes of a 2-byte sequence equivalent to code points < 0x7F
+        invalid_2B_seq_start_bytes = [bytes([x]) for x in range(0xC0, 0xC2)]
+        # start bytes of a 4-byte sequence equivalent to code points > 0x10FFFF
+        invalid_4B_seq_start_bytes = [bytes([x]) for x in range(0xF5, 0xF8)]
+        invalid_start_bytes = (
+            continuation_bytes + invalid_2B_seq_start_bytes +
+            invalid_4B_seq_start_bytes + [bytes([x]) for x in range(0xF7, 0x100)]
+        )
+
+        for byte in invalid_start_bytes:
+            self.assertRaises(UnicodeDecodeError, byte.decode, 'utf-8')
+
+        for sb in invalid_2B_seq_start_bytes:
+            for cb in continuation_bytes:
+                self.assertRaises(UnicodeDecodeError, (sb+cb).decode, 'utf-8')
+
+        for sb in invalid_4B_seq_start_bytes:
+            for cb1 in continuation_bytes[:3]:
+                for cb3 in continuation_bytes[:3]:
+                    self.assertRaises(UnicodeDecodeError,
+                                      (sb+cb1+b'\x80'+cb3).decode, 'utf-8')
+
+        for cb in [bytes([x]) for x in range(0x80, 0xA0)]:
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xE0'+cb+b'\x80').decode, 'utf-8')
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xE0'+cb+b'\xBF').decode, 'utf-8')
+        # surrogates
+        for cb in [bytes([x]) for x in range(0xA0, 0xC0)]:
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xED'+cb+b'\x80').decode, 'utf-8')
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xED'+cb+b'\xBF').decode, 'utf-8')
+        for cb in [bytes([x]) for x in range(0x80, 0x90)]:
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xF0'+cb+b'\x80\x80').decode, 'utf-8')
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xF0'+cb+b'\xBF\xBF').decode, 'utf-8')
+        for cb in [bytes([x]) for x in range(0x90, 0xC0)]:
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xF4'+cb+b'\x80\x80').decode, 'utf-8')
+            self.assertRaises(UnicodeDecodeError,
+                              (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8')
+
+    def test_issue8271(self):
+        # Issue #8271: during the decoding of an invalid UTF-8 byte sequence,
+        # only the start byte and the continuation byte(s) are now considered
+        # invalid, instead of the number of bytes specified by the start byte.
+        # See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf (page 95,
+        # table 3-8, Row 2) for more information about the algorithm used.
+        FFFD = '\ufffd'
+        sequences = [
+            # invalid start bytes
+            (b'\x80', FFFD), # continuation byte
+            (b'\x80\x80', FFFD*2), # 2 continuation bytes
+            (b'\xc0', FFFD),
+            (b'\xc0\xc0', FFFD*2),
+            (b'\xc1', FFFD),
+            (b'\xc1\xc0', FFFD*2),
+            (b'\xc0\xc1', FFFD*2),
+            # with start byte of a 2-byte sequence
+            (b'\xc2', FFFD), # only the start byte
+            (b'\xc2\xc2', FFFD*2), # 2 start bytes
+            (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes
+            (b'\xc2\x41', FFFD+'A'), # invalid continuation byte
+            # with start byte of a 3-byte sequence
+            (b'\xe1', FFFD), # only the start byte
+            (b'\xe1\xe1', FFFD*2), # 2 start bytes
+            (b'\xe1\xe1\xe1', FFFD*3), # 3 start bytes
+            (b'\xe1\xe1\xe1\xe1', FFFD*4), # 4 start bytes
+            (b'\xe1\x80', FFFD), # only 1 continuation byte
+            (b'\xe1\x41', FFFD+'A'), # invalid continuation byte
+            (b'\xe1\x41\x80', FFFD+'A'+FFFD), # invalid cb followed by valid cb
+            (b'\xe1\x41\x41', FFFD+'AA'), # 2 invalid continuation bytes
+            (b'\xe1\x80\x41', FFFD+'A'), # only 1 valid continuation byte
+            (b'\xe1\x80\xe1\x41', FFFD*2+'A'), # 1 valid and the other invalid
+            (b'\xe1\x41\xe1\x80', FFFD+'A'+FFFD), # 1 invalid and the other valid
+            # with start byte of a 4-byte sequence
+            (b'\xf1', FFFD), # only the start byte
+            (b'\xf1\xf1', FFFD*2), # 2 start bytes
+            (b'\xf1\xf1\xf1', FFFD*3), # 3 start bytes
+            (b'\xf1\xf1\xf1\xf1', FFFD*4), # 4 start bytes
+            (b'\xf1\xf1\xf1\xf1\xf1', FFFD*5), # 5 start bytes
+            (b'\xf1\x80', FFFD), # only 1 continuation bytes
+            (b'\xf1\x80\x80', FFFD), # only 2 continuation bytes
+            (b'\xf1\x80\x41', FFFD+'A'), # 1 valid cb and 1 invalid
+            (b'\xf1\x80\x41\x41', FFFD+'AA'), # 1 valid cb and 1 invalid
+            (b'\xf1\x80\x80\x41', FFFD+'A'), # 2 valid cb and 1 invalid
+            (b'\xf1\x41\x80', FFFD+'A'+FFFD), # 1 invalid cv and 1 valid
+            (b'\xf1\x41\x80\x80', FFFD+'A'+FFFD*2), # 1 invalid cb and 2 invalid
+            (b'\xf1\x41\x80\x41', FFFD+'A'+FFFD+'A'), # 2 invalid cb and 1 invalid
+            (b'\xf1\x41\x41\x80', FFFD+'AA'+FFFD), # 1 valid cb and 1 invalid
+            (b'\xf1\x41\xf1\x80', FFFD+'A'+FFFD),
+            (b'\xf1\x41\x80\xf1', FFFD+'A'+FFFD*2),
+            (b'\xf1\xf1\x80\x41', FFFD*2+'A'),
+            (b'\xf1\x41\xf1\xf1', FFFD+'A'+FFFD*2),
+            # with invalid start byte of a 4-byte sequence (rfc2279)
+            (b'\xf5', FFFD), # only the start byte
+            (b'\xf5\xf5', FFFD*2), # 2 start bytes
+            (b'\xf5\x80', FFFD*2), # only 1 continuation byte
+            (b'\xf5\x80\x80', FFFD*3), # only 2 continuation byte
+            (b'\xf5\x80\x80\x80', FFFD*4), # 3 continuation bytes
+            (b'\xf5\x80\x41', FFFD*2+'A'), #  1 valid cb and 1 invalid
+            (b'\xf5\x80\x41\xf5', FFFD*2+'A'+FFFD),
+            (b'\xf5\x41\x80\x80\x41', FFFD+'A'+FFFD*2+'A'),
+            # with invalid start byte of a 5-byte sequence (rfc2279)
+            (b'\xf8', FFFD), # only the start byte
+            (b'\xf8\xf8', FFFD*2), # 2 start bytes
+            (b'\xf8\x80', FFFD*2), # only one continuation byte
+            (b'\xf8\x80\x41', FFFD*2 + 'A'), # 1 valid cb and 1 invalid
+            (b'\xf8\x80\x80\x80\x80', FFFD*5), # invalid 5 bytes seq with 5 bytes
+            # with invalid start byte of a 6-byte sequence (rfc2279)
+            (b'\xfc', FFFD), # only the start byte
+            (b'\xfc\xfc', FFFD*2), # 2 start bytes
+            (b'\xfc\x80\x80', FFFD*3), # only 2 continuation bytes
+            (b'\xfc\x80\x80\x80\x80\x80', FFFD*6), # 6 continuation bytes
+            # invalid start byte
+            (b'\xfe', FFFD),
+            (b'\xfe\x80\x80', FFFD*3),
+            # other sequences
+            (b'\xf1\x80\x41\x42\x43', '\ufffd\x41\x42\x43'),
+            (b'\xf1\x80\xff\x42\x43', '\ufffd\ufffd\x42\x43'),
+            (b'\xf1\x80\xc2\x81\x43', '\ufffd\x81\x43'),
+            (b'\x61\xF1\x80\x80\xE1\x80\xC2\x62\x80\x63\x80\xBF\x64',
+             '\x61\uFFFD\uFFFD\uFFFD\x62\uFFFD\x63\uFFFD\uFFFD\x64'),
+        ]
+        for n, (seq, res) in enumerate(sequences):
+            self.assertRaises(UnicodeDecodeError, seq.decode, 'utf-8', 'strict')
+            self.assertEqual(seq.decode('utf-8', 'replace'), res)
+            self.assertEqual((seq+b'b').decode('utf-8', 'replace'), res+'b')
+            self.assertEqual(seq.decode('utf-8', 'ignore'),
+                             res.replace('\uFFFD', ''))
+
+    def assertCorrectUTF8Decoding(self, seq, res, err):
+        """
+        Check that an invalid UTF-8 sequence raises a UnicodeDecodeError when
+        'strict' is used, returns res when 'replace' is used, and that doesn't
+        return anything when 'ignore' is used.
+        """
+        with self.assertRaises(UnicodeDecodeError) as cm:
+            seq.decode('utf-8')
+        exc = cm.exception
+
+        self.assertIn(err, str(exc))
+        self.assertEqual(seq.decode('utf-8', 'replace'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'),
+                         'aaaa' + res + 'bbbb')
+        res = res.replace('\ufffd', '')
+        self.assertEqual(seq.decode('utf-8', 'ignore'), res)
+        self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'),
+                          'aaaa' + res + 'bbbb')
+
+    def test_invalid_start_byte(self):
+        """
+        Test that an 'invalid start byte' error is raised when the first byte
+        is not in the ASCII range or is not a valid start byte of a 2-, 3-, or
+        4-bytes sequence. The invalid start byte is replaced with a single
+        U+FFFD when errors='replace'.
+        E.g. <80> is a continuation byte and can appear only after a start byte.
+        """
+        FFFD = '\ufffd'
+        for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
+            self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd',
+                                           'invalid start byte')
+
+    def test_unexpected_end_of_data(self):
+        """
+        Test that an 'unexpected end of data' error is raised when the string
+        ends after a start byte of a 2-, 3-, or 4-bytes sequence without having
+        enough continuation bytes.  The incomplete sequence is replaced with a
+        single U+FFFD when errors='replace'.
+        E.g. in the sequence , F3 is the start byte of a 4-bytes
+        sequence, but it's followed by only 2 valid continuation bytes and the
+        last continuation bytes is missing.
+        Note: the continuation bytes must be all valid, if one of them is
+        invalid another error will be raised.
+        """
+        sequences = [
+            'C2', 'DF',
+            'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF',
+            'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF',
+            'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF',
+            'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF',
+            'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF',
+            'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
+        ]
+        FFFD = '\ufffd'
+        for seq in sequences:
+            self.assertCorrectUTF8Decoding(bytes.fromhex(seq), '\ufffd',
+                                           'unexpected end of data')
+
+    def test_invalid_cb_for_2bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte of a 2-bytes sequence is invalid.  The start byte
+        is replaced by a single U+FFFD and the second byte is handled
+        separately when errors='replace'.
+        E.g. in the sequence , C2 is the start byte of a 2-bytes
+        sequence, but 41 is not a valid continuation byte because it's the
+        ASCII letter 'A'.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'),
+            ('C2 C0', FFFDx2), ('C2 FF', FFFDx2),
+            ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'),
+            ('DF C0', FFFDx2), ('DF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_3bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 3-bytes sequence are invalid.  When
+        errors='replace', if the first continuation byte is valid, the first
+        two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the
+        third byte is handled separately, otherwise only the start byte is
+        replaced with a U+FFFD and the other continuation bytes are handled
+        separately.
+        E.g. in the sequence , E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        Python 2 used to consider all the bytes in range 80..BF valid when the
+        start byte was ED.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2),
+            ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2),
+            ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'),
+            ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2),
+            ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'),
+            ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'),
+            ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2),
+            ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'),
+            ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2),
+            ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'),
+            ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'),
+            ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2),
+            ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'),
+            ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2),
+            ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'),
+            ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'),
+            ('ED 7F', FFFD+'\x7f'),
+            ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^
+            ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'),
+            ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2),
+            ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'),
+            ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2),
+            ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'),
+            ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2),
+            ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'),
+            ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2),
+            ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'),
+            ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'),
+            ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2),
+            ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'),
+            ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2),
+            ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'),
+            ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2),
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res,
+                                           'invalid continuation byte')
+
+    def test_invalid_cb_for_4bytes_seq(self):
+        """
+        Test that an 'invalid continuation byte' error is raised when the
+        continuation byte(s) of a 4-bytes sequence are invalid.  When
+        errors='replace',the start byte and all the following valid
+        continuation bytes are replaced with a single U+FFFD, and all the bytes
+        starting from the first invalid continuation bytes (included) are
+        handled separately.
+        E.g. in the sequence , E1 is the start byte of a 3-bytes
+        sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+        because it's the ASCII letter 'A'.
+        Note: when the start byte is E0 or ED, the valid ranges for the first
+        continuation byte are limited to A0..BF and 80..9F respectively.
+        However, when the start byte is ED, Python 2 considers all the bytes
+        in range 80..BF valid.  This is fixed in Python 3.
+        """
+        FFFD = '\ufffd'
+        FFFDx2 = FFFD * 2
+        sequences = [
+            ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2),
+            ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2),
+            ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'),
+            ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2),
+            ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'),
+            ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2),
+            ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'),
+            ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2),
+            ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'),
+            ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2),
+            ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'),
+            ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2),
+            ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'),
+            ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2),
+            ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2),
+            ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'),
+            ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2),
+            ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'),
+            ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2),
+            ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'),
+            ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2),
+            ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'),
+            ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2),
+            ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'),
+            ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2),
+            ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'),
+            ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2),
+            ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'),
+            ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2),
+            ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'),
+            ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2),
+            ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'),
+            ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2),
+            ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'),
+            ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2),
+            ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'),
+            ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2),
+            ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'),
+            ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2),
+            ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'),
+            ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2),
+            ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2),
+            ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2),
+            ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'),
+            ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2),
+            ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'),
+            ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2),
+            ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'),
+            ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2),
+            ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'),
+            ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2),
+            ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'),
+            ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2),
+            ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'),
+            ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2)
+        ]
+        for seq, res in sequences:
+            self.assertCorrectUTF8Decoding(bytes.fromhex(seq), res,
+                                           'invalid continuation byte')
+
+    def test_codecs_idna(self):
+        # Test whether trailing dot is preserved
+        self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")
+
+    def test_codecs_errors(self):
+        # Error handling (encoding)
+        self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii')
+        self.assertRaises(UnicodeError, 'Andr\202 x'.encode, 'ascii','strict')
+        self.assertEqual('Andr\202 x'.encode('ascii','ignore'), b"Andr x")
+        self.assertEqual('Andr\202 x'.encode('ascii','replace'), b"Andr? x")
+        self.assertEqual('Andr\202 x'.encode('ascii', 'replace'),
+                         'Andr\202 x'.encode('ascii', errors='replace'))
+        self.assertEqual('Andr\202 x'.encode('ascii', 'ignore'),
+                         'Andr\202 x'.encode(encoding='ascii', errors='ignore'))
+
+        # Error handling (decoding)
+        self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii')
+        self.assertRaises(UnicodeError, str, b'Andr\202 x', 'ascii', 'strict')
+        self.assertEqual(str(b'Andr\202 x', 'ascii', 'ignore'), "Andr x")
+        self.assertEqual(str(b'Andr\202 x', 'ascii', 'replace'), 'Andr\uFFFD x')
+        self.assertEqual(str(b'\202 x', 'ascii', 'replace'), '\uFFFD x')
+
+        # Error handling (unknown character names)
+        self.assertEqual(b"\\N{foo}xx".decode("unicode-escape", "ignore"), "xx")
+
+        # Error handling (truncated escape sequence)
+        self.assertRaises(UnicodeError, b"\\".decode, "unicode-escape")
+
+        self.assertRaises(TypeError, b"hello".decode, "test.unicode1")
+        self.assertRaises(TypeError, str, b"hello", "test.unicode2")
+        self.assertRaises(TypeError, "hello".encode, "test.unicode1")
+        self.assertRaises(TypeError, "hello".encode, "test.unicode2")
+
+        # Error handling (wrong arguments)
+        self.assertRaises(TypeError, "hello".encode, 42, 42, 42)
+
+        # Error handling (lone surrogate in
+        # _PyUnicode_TransformDecimalAndSpaceToASCII())
+        self.assertRaises(ValueError, int, "\ud800")
+        self.assertRaises(ValueError, int, "\udf00")
+        self.assertRaises(ValueError, float, "\ud800")
+        self.assertRaises(ValueError, float, "\udf00")
+        self.assertRaises(ValueError, complex, "\ud800")
+        self.assertRaises(ValueError, complex, "\udf00")
+
+    def test_codecs(self):
+        # Encoding
+        self.assertEqual('hello'.encode('ascii'), b'hello')
+        self.assertEqual('hello'.encode('utf-7'), b'hello')
+        self.assertEqual('hello'.encode('utf-8'), b'hello')
+        self.assertEqual('hello'.encode('utf-8'), b'hello')
+        self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000')
+        self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o')
+        self.assertEqual('hello'.encode('latin-1'), b'hello')
+
+        # Default encoding is utf-8
+        self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83')
+
+        # Roundtrip safety for BMP (just the first 1024 chars)
+        for c in range(1024):
+            u = chr(c)
+            for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le',
+                             'utf-16-be', 'raw_unicode_escape',
+                             'unicode_escape'):
+                self.assertEqual(str(u.encode(encoding),encoding), u)
+
+        # Roundtrip safety for BMP (just the first 256 chars)
+        for c in range(256):
+            u = chr(c)
+            for encoding in ('latin-1',):
+                self.assertEqual(str(u.encode(encoding),encoding), u)
+
+        # Roundtrip safety for BMP (just the first 128 chars)
+        for c in range(128):
+            u = chr(c)
+            for encoding in ('ascii',):
+                self.assertEqual(str(u.encode(encoding),encoding), u)
+
+        # Roundtrip safety for non-BMP (just a few chars)
+        with warnings.catch_warnings():
+            u = '\U00010001\U00020002\U00030003\U00040004\U00050005'
+            for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
+                             'raw_unicode_escape', 'unicode_escape'):
+                self.assertEqual(str(u.encode(encoding),encoding), u)
+
+        # UTF-8 must be roundtrip safe for all code points
+        # (except surrogates, which are forbidden).
+        u = ''.join(map(chr, list(range(0, 0xd800)) +
+                             list(range(0xe000, 0x110000))))
+        for encoding in ('utf-8',):
+            self.assertEqual(str(u.encode(encoding),encoding), u)
+
+    @unittest.skipIf(sys.version_info < (3, 5), 'codecs test requires Py3.5+')
+    def test_codecs_charmap(self):
+        # 0-127
+        s = bytes(range(128))
+        for encoding in (
+            'cp037', 'cp1026', 'cp273',
+            'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850',
+            'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862',
+            'cp863', 'cp865', 'cp866', 'cp1125',
+            'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+            'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
+            'iso8859_7', 'iso8859_9',
+            'koi8_r', 'koi8_t', 'koi8_u', 'kz1048', 'latin_1',
+            'mac_cyrillic', 'mac_latin2',
+
+            'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+            'cp1256', 'cp1257', 'cp1258',
+            'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+
+            'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+            'cp1006', 'iso8859_8',
+
+            ### These have undefined mappings:
+            #'cp424',
+
+            ### These fail the round-trip:
+            #'cp875'
+
+            ):
+            self.assertEqual(str(s, encoding).encode(encoding), s)
+
+        # 128-255
+        s = bytes(range(128, 256))
+        for encoding in (
+            'cp037', 'cp1026', 'cp273',
+            'cp437', 'cp500', 'cp720', 'cp737', 'cp775', 'cp850',
+            'cp852', 'cp855', 'cp858', 'cp860', 'cp861', 'cp862',
+            'cp863', 'cp865', 'cp866', 'cp1125',
+            'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+            'iso8859_2', 'iso8859_4', 'iso8859_5',
+            'iso8859_9', 'koi8_r', 'koi8_u', 'latin_1',
+            'mac_cyrillic', 'mac_latin2',
+
+            ### These have undefined mappings:
+            #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+            #'cp1256', 'cp1257', 'cp1258',
+            #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+            #'iso8859_3', 'iso8859_6', 'iso8859_7', 'koi8_t', 'kz1048',
+            #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+
+            ### These fail the round-trip:
+            #'cp1006', 'cp875', 'iso8859_8',
+
+            ):
+            self.assertEqual(str(s, encoding).encode(encoding), s)
+
+    def test_concatenation(self):
+        self.assertEqual(("abc" "def"), "abcdef")
+        self.assertEqual(("abc" "def"), "abcdef")
+        self.assertEqual(("abc" "def"), "abcdef")
+        self.assertEqual(("abc" "def" "ghi"), "abcdefghi")
+        self.assertEqual(("abc" "def" "ghi"), "abcdefghi")
+
+    def test_printing(self):
+        class BitBucket:
+            def write(self, text):
+                pass
+
+        out = BitBucket()
+        print('abc', file=out)
+        print('abc', 'def', file=out)
+        print('abc', 'def', file=out)
+        print('abc', 'def', file=out)
+        print('abc\n', file=out)
+        print('abc\n', end=' ', file=out)
+        print('abc\n', end=' ', file=out)
+        print('def\n', file=out)
+        print('def\n', file=out)
+
+    def test_ucs4(self):
+        x = '\U00100000'
+        y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
+        self.assertEqual(x, y)
+
+        y = br'\U00100000'
+        x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
+        self.assertEqual(x, y)
+        y = br'\U00010000'
+        x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
+        self.assertEqual(x, y)
+
+        try:
+            br'\U11111111'.decode("raw-unicode-escape")
+        except UnicodeDecodeError as e:
+            self.assertEqual(e.start, 0)
+            self.assertEqual(e.end, 10)
+        else:
+            self.fail("Should have raised UnicodeDecodeError")
+
+    def test_conversion(self):
+        # Make sure __str__() works properly
+        class ObjectToStr:
+            def __str__(self):
+                return "foo"
+
+        class StrSubclassToStr(str):
+            def __str__(self):
+                return "foo"
+
+        class StrSubclassToStrSubclass(str):
+            def __new__(cls, content=""):
+                return str.__new__(cls, 2*content)
+            def __str__(self):
+                return self
+
+        self.assertEqual(str(ObjectToStr()), "foo")
+        self.assertEqual(str(StrSubclassToStr("bar")), "foo")
+        s = str(StrSubclassToStrSubclass("foo"))
+        self.assertEqual(s, "foofoo")
+        self.assertIs(type(s), StrSubclassToStrSubclass)
+        s = StrSubclass(StrSubclassToStrSubclass("foo"))
+        self.assertEqual(s, "foofoo")
+        self.assertIs(type(s), StrSubclass)
+
+    def test_unicode_repr(self):
+        class s1:
+            def __repr__(self):
+                return '\\n'
+
+        class s2:
+            def __repr__(self):
+                return '\\n'
+
+        self.assertEqual(repr(s1()), '\\n')
+        self.assertEqual(repr(s2()), '\\n')
+
+    def test_printable_repr(self):
+        self.assertEqual(repr('\U00010000'), "'%c'" % (0x10000,)) # printable
+        self.assertEqual(repr('\U00014000'), "'\\U00014000'")     # nonprintable
+
+    # This test only affects 32-bit platforms because expandtabs can only take
+    # an int as the max value, not a 64-bit C long.  If expandtabs is changed
+    # to take a 64-bit long, this test should apply to all platforms.
+    @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4,
+                     'only applies to 32-bit platforms')
+    def test_expandtabs_overflows_gracefully(self):
+        self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize)
+
+    @support.cpython_only
+    def test_expandtabs_optimization(self):
+        s = 'abc'
+        self.assertIs(s.expandtabs(), s)
+
+    def test_raiseMemError(self):
+        if struct.calcsize('P') == 8:
+            # 64 bits pointers
+            ascii_struct_size = 48
+            compact_struct_size = 72
+        else:
+            # 32 bits pointers
+            ascii_struct_size = 24
+            compact_struct_size = 36
+
+        for char in ('a', '\xe9', '\u20ac', '\U0010ffff'):
+            code = ord(char)
+            if code < 0x100:
+                char_size = 1  # sizeof(Py_UCS1)
+                struct_size = ascii_struct_size
+            elif code < 0x10000:
+                char_size = 2  # sizeof(Py_UCS2)
+                struct_size = compact_struct_size
+            else:
+                char_size = 4  # sizeof(Py_UCS4)
+                struct_size = compact_struct_size
+            # Note: sys.maxsize is half of the actual max allocation because of
+            # the signedness of Py_ssize_t. Strings of maxlen-1 should in principle
+            # be allocatable, given enough memory.
+            maxlen = ((sys.maxsize - struct_size) // char_size)
+            alloc = lambda: char * maxlen
+            self.assertRaises(MemoryError, alloc)
+            self.assertRaises(MemoryError, alloc)
+
+    def test_format_subclass(self):
+        class S(str):
+            def __str__(self):
+                return '__str__ overridden'
+        s = S('xxx')
+        self.assertEqual("%s" % s, '__str__ overridden')
+        self.assertEqual("{}".format(s), '__str__ overridden')
+
+    def test_subclass_add(self):
+        class S(str):
+            def __add__(self, o):
+                return "3"
+        self.assertEqual(S("4") + S("5"), "3")
+        class S(str):
+            def __iadd__(self, o):
+                return "3"
+        s = S("1")
+        s += "4"
+        self.assertEqual(s, "3")
+
+    def _test_getnewargs(self):
+        text = 'abc'
+        args = text.__getnewargs__()
+        self.assertIsNot(args[0], text)
+        self.assertEqual(args[0], text)
+        self.assertEqual(len(args), 1)
+
+    @unittest.skipIf(sys.version_info < (3, 8), 'resize test requires Py3.8+')
+    @support.cpython_only
+    def test_resize(self):
+        from _testcapi import getargs_u
+        for length in range(1, 100, 7):
+            # generate a fresh string (refcount=1)
+            text = 'a' * length + 'b'
+
+            # fill wstr internal field
+            abc = getargs_u(text)
+            self.assertEqual(abc, text)
+
+            # resize text: wstr field must be cleared and then recomputed
+            text += 'c'
+            abcdef = getargs_u(text)
+            self.assertNotEqual(abc, abcdef)
+            self.assertEqual(abcdef, text)
+
+    def test_compare(self):
+        # Issue #17615
+        N = 10
+        ascii = 'a' * N
+        ascii2 = 'z' * N
+        latin = '\x80' * N
+        latin2 = '\xff' * N
+        bmp = '\u0100' * N
+        bmp2 = '\uffff' * N
+        astral = '\U00100000' * N
+        astral2 = '\U0010ffff' * N
+        strings = (
+            ascii, ascii2,
+            latin, latin2,
+            bmp, bmp2,
+            astral, astral2)
+        for text1, text2 in itertools.combinations(strings, 2):
+            equal = (text1 is text2)
+            self.assertEqual(text1 == text2, equal)
+            self.assertEqual(text1 != text2, not equal)
+
+            if equal:
+                self.assertTrue(text1 <= text2)
+                self.assertTrue(text1 >= text2)
+
+                # text1 is text2: duplicate strings to skip the "str1 == str2"
+                # optimization in unicode_compare_eq() and really compare
+                # character per character
+                copy1 = duplicate_string(text1)
+                copy2 = duplicate_string(text2)
+                self.assertIsNot(copy1, copy2)
+
+                self.assertTrue(copy1 == copy2)
+                self.assertFalse(copy1 != copy2)
+
+                self.assertTrue(copy1 <= copy2)
+                self.assertTrue(copy2 >= copy2)
+
+        self.assertTrue(ascii < ascii2)
+        self.assertTrue(ascii < latin)
+        self.assertTrue(ascii < bmp)
+        self.assertTrue(ascii < astral)
+        self.assertFalse(ascii >= ascii2)
+        self.assertFalse(ascii >= latin)
+        self.assertFalse(ascii >= bmp)
+        self.assertFalse(ascii >= astral)
+
+        self.assertFalse(latin < ascii)
+        self.assertTrue(latin < latin2)
+        self.assertTrue(latin < bmp)
+        self.assertTrue(latin < astral)
+        self.assertTrue(latin >= ascii)
+        self.assertFalse(latin >= latin2)
+        self.assertFalse(latin >= bmp)
+        self.assertFalse(latin >= astral)
+
+        self.assertFalse(bmp < ascii)
+        self.assertFalse(bmp < latin)
+        self.assertTrue(bmp < bmp2)
+        self.assertTrue(bmp < astral)
+        self.assertTrue(bmp >= ascii)
+        self.assertTrue(bmp >= latin)
+        self.assertFalse(bmp >= bmp2)
+        self.assertFalse(bmp >= astral)
+
+        self.assertFalse(astral < ascii)
+        self.assertFalse(astral < latin)
+        self.assertFalse(astral < bmp2)
+        self.assertTrue(astral < astral2)
+        self.assertTrue(astral >= ascii)
+        self.assertTrue(astral >= latin)
+        self.assertTrue(astral >= bmp2)
+        self.assertFalse(astral >= astral2)
+
+    def test_free_after_iterating(self):
+        support.check_free_after_iterating(self, iter, str)
+        support.check_free_after_iterating(self, reversed, str)
+
+
+u"""
+class CAPITest(unittest.TestCase):
+
+    # Test PyUnicode_FromFormat()
+    def test_from_format(self):
+        support.import_module('ctypes')
+        from ctypes import (
+            pythonapi, py_object, sizeof,
+            c_int, c_long, c_longlong, c_ssize_t,
+            c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
+        name = "PyUnicode_FromFormat"
+        _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.restype = py_object
+
+        def PyUnicode_FromFormat(format, *args):
+            cargs = tuple(
+                py_object(arg) if isinstance(arg, str) else arg
+                for arg in args)
+            return _PyUnicode_FromFormat(format, *cargs)
+
+        def check_format(expected, format, *args):
+            text = PyUnicode_FromFormat(format, *args)
+            self.assertEqual(expected, text)
+
+        # ascii format, non-ascii argument
+        check_format('ascii\x7f=unicode\xe9',
+                     b'ascii\x7f=%U', 'unicode\xe9')
+
+        # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV()
+        # raises an error
+        self.assertRaisesRegex(ValueError,
+            r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format '
+            'string, got a non-ASCII byte: 0xe9$',
+            PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii')
+
+        # test "%c"
+        check_format('\uabcd',
+                     b'%c', c_int(0xabcd))
+        check_format('\U0010ffff',
+                     b'%c', c_int(0x10ffff))
+        with self.assertRaises(OverflowError):
+            PyUnicode_FromFormat(b'%c', c_int(0x110000))
+        # Issue #18183
+        check_format('\U00010000\U00100000',
+                     b'%c%c', c_int(0x10000), c_int(0x100000))
+
+        # test "%"
+        check_format('%',
+                     b'%')
+        check_format('%',
+                     b'%%')
+        check_format('%s',
+                     b'%%s')
+        check_format('[%]',
+                     b'[%%]')
+        check_format('%abc',
+                     b'%%%s', b'abc')
+
+        # truncated string
+        check_format('abc',
+                     b'%.3s', b'abcdef')
+        check_format('abc[\ufffd',
+                     b'%.5s', 'abc[\u20ac]'.encode('utf8'))
+        check_format("'\\u20acABC'",
+                     b'%A', '\u20acABC')
+        check_format("'\\u20",
+                     b'%.5A', '\u20acABCDEF')
+        check_format("'\u20acABC'",
+                     b'%R', '\u20acABC')
+        check_format("'\u20acA",
+                     b'%.3R', '\u20acABCDEF')
+        check_format('\u20acAB',
+                     b'%.3S', '\u20acABCDEF')
+        check_format('\u20acAB',
+                     b'%.3U', '\u20acABCDEF')
+        check_format('\u20acAB',
+                     b'%.3V', '\u20acABCDEF', None)
+        check_format('abc[\ufffd',
+                     b'%.5V', None, 'abc[\u20ac]'.encode('utf8'))
+
+        # following tests comes from #7330
+        # test width modifier and precision modifier with %S
+        check_format("repr=  abc",
+                     b'repr=%5S', 'abc')
+        check_format("repr=ab",
+                     b'repr=%.2S', 'abc')
+        check_format("repr=   ab",
+                     b'repr=%5.2S', 'abc')
+
+        # test width modifier and precision modifier with %R
+        check_format("repr=   'abc'",
+                     b'repr=%8R', 'abc')
+        check_format("repr='ab",
+                     b'repr=%.3R', 'abc')
+        check_format("repr=  'ab",
+                     b'repr=%5.3R', 'abc')
+
+        # test width modifier and precision modifier with %A
+        check_format("repr=   'abc'",
+                     b'repr=%8A', 'abc')
+        check_format("repr='ab",
+                     b'repr=%.3A', 'abc')
+        check_format("repr=  'ab",
+                     b'repr=%5.3A', 'abc')
+
+        # test width modifier and precision modifier with %s
+        check_format("repr=  abc",
+                     b'repr=%5s', b'abc')
+        check_format("repr=ab",
+                     b'repr=%.2s', b'abc')
+        check_format("repr=   ab",
+                     b'repr=%5.2s', b'abc')
+
+        # test width modifier and precision modifier with %U
+        check_format("repr=  abc",
+                     b'repr=%5U', 'abc')
+        check_format("repr=ab",
+                     b'repr=%.2U', 'abc')
+        check_format("repr=   ab",
+                     b'repr=%5.2U', 'abc')
+
+        # test width modifier and precision modifier with %V
+        check_format("repr=  abc",
+                     b'repr=%5V', 'abc', b'123')
+        check_format("repr=ab",
+                     b'repr=%.2V', 'abc', b'123')
+        check_format("repr=   ab",
+                     b'repr=%5.2V', 'abc', b'123')
+        check_format("repr=  123",
+                     b'repr=%5V', None, b'123')
+        check_format("repr=12",
+                     b'repr=%.2V', None, b'123')
+        check_format("repr=   12",
+                     b'repr=%5.2V', None, b'123')
+
+        # test integer formats (%i, %d, %u)
+        check_format('010',
+                     b'%03i', c_int(10))
+        check_format('0010',
+                     b'%0.4i', c_int(10))
+        check_format('-123',
+                     b'%i', c_int(-123))
+        check_format('-123',
+                     b'%li', c_long(-123))
+        check_format('-123',
+                     b'%lli', c_longlong(-123))
+        check_format('-123',
+                     b'%zi', c_ssize_t(-123))
+
+        check_format('-123',
+                     b'%d', c_int(-123))
+        check_format('-123',
+                     b'%ld', c_long(-123))
+        check_format('-123',
+                     b'%lld', c_longlong(-123))
+        check_format('-123',
+                     b'%zd', c_ssize_t(-123))
+
+        check_format('123',
+                     b'%u', c_uint(123))
+        check_format('123',
+                     b'%lu', c_ulong(123))
+        check_format('123',
+                     b'%llu', c_ulonglong(123))
+        check_format('123',
+                     b'%zu', c_size_t(123))
+
+        # test long output
+        min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1))
+        max_longlong = -min_longlong - 1
+        check_format(str(min_longlong),
+                     b'%lld', c_longlong(min_longlong))
+        check_format(str(max_longlong),
+                     b'%lld', c_longlong(max_longlong))
+        max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1
+        check_format(str(max_ulonglong),
+                     b'%llu', c_ulonglong(max_ulonglong))
+        PyUnicode_FromFormat(b'%p', c_void_p(-1))
+
+        # test padding (width and/or precision)
+        check_format('123'.rjust(10, '0'),
+                     b'%010i', c_int(123))
+        check_format('123'.rjust(100),
+                     b'%100i', c_int(123))
+        check_format('123'.rjust(100, '0'),
+                     b'%.100i', c_int(123))
+        check_format('123'.rjust(80, '0').rjust(100),
+                     b'%100.80i', c_int(123))
+
+        check_format('123'.rjust(10, '0'),
+                     b'%010u', c_uint(123))
+        check_format('123'.rjust(100),
+                     b'%100u', c_uint(123))
+        check_format('123'.rjust(100, '0'),
+                     b'%.100u', c_uint(123))
+        check_format('123'.rjust(80, '0').rjust(100),
+                     b'%100.80u', c_uint(123))
+
+        check_format('123'.rjust(10, '0'),
+                     b'%010x', c_int(0x123))
+        check_format('123'.rjust(100),
+                     b'%100x', c_int(0x123))
+        check_format('123'.rjust(100, '0'),
+                     b'%.100x', c_int(0x123))
+        check_format('123'.rjust(80, '0').rjust(100),
+                     b'%100.80x', c_int(0x123))
+
+        # test %A
+        check_format(r"%A:'abc\xe9\uabcd\U0010ffff'",
+                     b'%%A:%A', 'abc\xe9\uabcd\U0010ffff')
+
+        # test %V
+        check_format('repr=abc',
+                     b'repr=%V', 'abc', b'xyz')
+
+        # Test string decode from parameter of %s using utf-8.
+        # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of
+        # '\u4eba\u6c11'
+        check_format('repr=\u4eba\u6c11',
+                     b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91')
+
+        #Test replace error handler.
+        check_format('repr=abc\ufffd',
+                     b'repr=%V', None, b'abc\xff')
+
+        # not supported: copy the raw format string. these tests are just here
+        # to check for crashes and should not be considered as specifications
+        check_format('%s',
+                     b'%1%s', b'abc')
+        check_format('%1abc',
+                     b'%1abc')
+        check_format('%+i',
+                     b'%+i', c_int(10))
+        check_format('%.%s',
+                     b'%.%s', b'abc')
+
+        # Issue #33817: empty strings
+        check_format('',
+                     b'')
+        check_format('',
+                     b'%s', b'')
+
+    # Test PyUnicode_AsWideChar()
+    @support.cpython_only
+    def test_aswidechar(self):
+        from _testcapi import unicode_aswidechar
+        support.import_module('ctypes')
+        from ctypes import c_wchar, sizeof
+
+        wchar, size = unicode_aswidechar('abcdef', 2)
+        self.assertEqual(size, 2)
+        self.assertEqual(wchar, 'ab')
+
+        wchar, size = unicode_aswidechar('abc', 3)
+        self.assertEqual(size, 3)
+        self.assertEqual(wchar, 'abc')
+
+        wchar, size = unicode_aswidechar('abc', 4)
+        self.assertEqual(size, 3)
+        self.assertEqual(wchar, 'abc\0')
+
+        wchar, size = unicode_aswidechar('abc', 10)
+        self.assertEqual(size, 3)
+        self.assertEqual(wchar, 'abc\0')
+
+        wchar, size = unicode_aswidechar('abc\0def', 20)
+        self.assertEqual(size, 7)
+        self.assertEqual(wchar, 'abc\0def\0')
+
+        nonbmp = chr(0x10ffff)
+        if sizeof(c_wchar) == 2:
+            buflen = 3
+            nchar = 2
+        else: # sizeof(c_wchar) == 4
+            buflen = 2
+            nchar = 1
+        wchar, size = unicode_aswidechar(nonbmp, buflen)
+        self.assertEqual(size, nchar)
+        self.assertEqual(wchar, nonbmp + '\0')
+
+    # Test PyUnicode_AsWideCharString()
+    @support.cpython_only
+    def test_aswidecharstring(self):
+        from _testcapi import unicode_aswidecharstring
+        support.import_module('ctypes')
+        from ctypes import c_wchar, sizeof
+
+        wchar, size = unicode_aswidecharstring('abc')
+        self.assertEqual(size, 3)
+        self.assertEqual(wchar, 'abc\0')
+
+        wchar, size = unicode_aswidecharstring('abc\0def')
+        self.assertEqual(size, 7)
+        self.assertEqual(wchar, 'abc\0def\0')
+
+        nonbmp = chr(0x10ffff)
+        if sizeof(c_wchar) == 2:
+            nchar = 2
+        else: # sizeof(c_wchar) == 4
+            nchar = 1
+        wchar, size = unicode_aswidecharstring(nonbmp)
+        self.assertEqual(size, nchar)
+        self.assertEqual(wchar, nonbmp + '\0')
+
+    # Test PyUnicode_AsUCS4()
+    @support.cpython_only
+    def test_asucs4(self):
+        from _testcapi import unicode_asucs4
+        for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
+                  'a\ud800b\udfffc', '\ud834\udd1e']:
+            l = len(s)
+            self.assertEqual(unicode_asucs4(s, l, 1), s+'\0')
+            self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff')
+            self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff')
+            self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff')
+            self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1)
+            self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0)
+            s = '\0'.join([s, s])
+            self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0')
+            self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff')
+
+    # Test PyUnicode_FindChar()
+    @support.cpython_only
+    def test_findchar(self):
+        from _testcapi import unicode_findchar
+
+        for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1":
+            for i, ch in enumerate(str):
+                self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i)
+                self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i)
+
+        str = "!>_= end
+        self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1)
+        self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1)
+        # negative
+        self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0)
+        self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0)
+
+    # Test PyUnicode_CopyCharacters()
+    @support.cpython_only
+    def test_copycharacters(self):
+        from _testcapi import unicode_copycharacters
+
+        strings = [
+            'abcde', '\xa1\xa2\xa3\xa4\xa5',
+            '\u4f60\u597d\u4e16\u754c\uff01',
+            '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604'
+        ]
+
+        for idx, from_ in enumerate(strings):
+            # wide -> narrow: exceed maxchar limitation
+            for to in strings[:idx]:
+                self.assertRaises(
+                    SystemError,
+                    unicode_copycharacters, to, 0, from_, 0, 5
+                )
+            # same kind
+            for from_start in range(5):
+                self.assertEqual(
+                    unicode_copycharacters(from_, 0, from_, from_start, 5),
+                    (from_[from_start:from_start+5].ljust(5, '\0'),
+                     5-from_start)
+                )
+            for to_start in range(5):
+                self.assertEqual(
+                    unicode_copycharacters(from_, to_start, from_, to_start, 5),
+                    (from_[to_start:to_start+5].rjust(5, '\0'),
+                     5-to_start)
+                )
+            # narrow -> wide
+            # Tests omitted since this creates invalid strings.
+
+        s = strings[0]
+        self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5)
+        self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5)
+        self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5)
+        self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5)
+        self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5)
+        self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1)
+        self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0)
+
+    @support.cpython_only
+    def test_encode_decimal(self):
+        from _testcapi import unicode_encodedecimal
+        self.assertEqual(unicode_encodedecimal('123'),
+                         b'123')
+        self.assertEqual(unicode_encodedecimal('\u0663.\u0661\u0664'),
+                         b'3.14')
+        self.assertEqual(unicode_encodedecimal("\N{EM SPACE}3.14\N{EN SPACE}"),
+                         b' 3.14 ')
+        self.assertRaises(UnicodeEncodeError,
+                          unicode_encodedecimal, "123\u20ac", "strict")
+        self.assertRaisesRegex(
+            ValueError,
+            "^'decimal' codec can't encode character",
+            unicode_encodedecimal, "123\u20ac", "replace")
+
+    @support.cpython_only
+    def test_transform_decimal(self):
+        from _testcapi import unicode_transformdecimaltoascii as transform_decimal
+        self.assertEqual(transform_decimal('123'),
+                         '123')
+        self.assertEqual(transform_decimal('\u0663.\u0661\u0664'),
+                         '3.14')
+        self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"),
+                         "\N{EM SPACE}3.14\N{EN SPACE}")
+        self.assertEqual(transform_decimal('123\u20ac'),
+                         '123\u20ac')
+
+    @support.cpython_only
+    def test_pep393_utf8_caching_bug(self):
+        # Issue #25709: Problem with string concatenation and utf-8 cache
+        from _testcapi import getargs_s_hash
+        for k in 0x24, 0xa4, 0x20ac, 0x1f40d:
+            s = ''
+            for i in range(5):
+                # Due to CPython specific optimization the 's' string can be
+                # resized in-place.
+                s += chr(k)
+                # Parsing with the "s#" format code calls indirectly
+                # PyUnicode_AsUTF8AndSize() which creates the UTF-8
+                # encoded string cached in the Unicode object.
+                self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
+                # Check that the second call returns the same result
+                self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
+"""
+
+
+u"""
+class StringModuleTest(unittest.TestCase):
+    def test_formatter_parser(self):
+        def parse(format):
+            return list(_string.formatter_parser(format))
+
+        formatter = parse("prefix {2!s}xxx{0:^+10.3f}{obj.attr!s} {z[0]!s:10}")
+        self.assertEqual(formatter, [
+            ('prefix ', '2', '', 's'),
+            ('xxx', '0', '^+10.3f', None),
+            ('', 'obj.attr', '', 's'),
+            (' ', 'z[0]', '10', 's'),
+        ])
+
+        formatter = parse("prefix {} suffix")
+        self.assertEqual(formatter, [
+            ('prefix ', '', '', None),
+            (' suffix', None, None, None),
+        ])
+
+        formatter = parse("str")
+        self.assertEqual(formatter, [
+            ('str', None, None, None),
+        ])
+
+        formatter = parse("")
+        self.assertEqual(formatter, [])
+
+        formatter = parse("{0}")
+        self.assertEqual(formatter, [
+            ('', '0', '', None),
+        ])
+
+        self.assertRaises(TypeError, _string.formatter_parser, 1)
+
+    def test_formatter_field_name_split(self):
+        def split(name):
+            items = list(_string.formatter_field_name_split(name))
+            items[1] = list(items[1])
+            return items
+        self.assertEqual(split("obj"), ["obj", []])
+        self.assertEqual(split("obj.arg"), ["obj", [(True, 'arg')]])
+        self.assertEqual(split("obj[key]"), ["obj", [(False, 'key')]])
+        self.assertEqual(split("obj.arg[key1][key2]"), [
+            "obj",
+            [(True, 'arg'),
+             (False, 'key1'),
+             (False, 'key2'),
+            ]])
+        self.assertRaises(TypeError, _string.formatter_field_name_split, 1)
+"""
+
+
+if __name__ == "__main__":
+    unittest.main()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/test_unicode_string_tests.pxi cython-0.20.1+1~202203241016-9537/tests/run/test_unicode_string_tests.pxi
--- cython-0.20.1+1~201611251650-6686/tests/run/test_unicode_string_tests.pxi	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/test_unicode_string_tests.pxi	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,1411 @@
+"""
+Common tests shared by test_unicode, test_userstring and test_bytes.
+"""
+
+import unittest, string, sys, struct
+#from test import support
+#from collections import UserList
+
+class Sequence:
+    def __init__(self, seq='wxyz'): self.seq = seq
+    def __len__(self): return len(self.seq)
+    def __getitem__(self, i): return self.seq[i]
+
+class BadSeq1(Sequence):
+    def __init__(self): self.seq = [7, 'hello', 123]
+    def __str__(self): return '{0} {1} {2}'.format(*self.seq)
+
+class BadSeq2(Sequence):
+    def __init__(self): self.seq = ['a', 'b', 'c']
+    def __len__(self): return 8
+
+class BaseTest:
+    # These tests are for buffers of values (bytes) and not
+    # specific to character interpretation, used for bytes objects
+    # and various string implementations
+
+    # The type to be tested
+    # Change in subclasses to change the behaviour of fixtesttype()
+    type2test = None
+
+    # Whether the "contained items" of the container are integers in
+    # range(0, 256) (i.e. bytes, bytearray) or strings of length 1
+    # (str)
+    contains_bytes = False
+
+    # All tests pass their arguments to the testing methods
+    # as str objects. fixtesttype() can be used to propagate
+    # these arguments to the appropriate type
+    def fixtype(self, obj):
+        if isinstance(obj, str):
+            return self.__class__.type2test(obj)
+        elif isinstance(obj, list):
+            return [self.fixtype(x) for x in obj]
+        elif isinstance(obj, tuple):
+            return tuple([self.fixtype(x) for x in obj])
+        elif isinstance(obj, dict):
+            return dict([
+               (self.fixtype(key), self.fixtype(value))
+               for (key, value) in obj.items()
+            ])
+        else:
+            return obj
+
+    def test_fixtype(self):
+        self.assertIs(type(self.fixtype("123")), self.type2test)
+
+    # check that obj.method(*args) returns result
+    def checkequal(self, result, obj, methodname, *args, **kwargs):
+        result = self.fixtype(result)
+        obj = self.fixtype(obj)
+        args = self.fixtype(args)
+        kwargs = {k: self.fixtype(v) for k,v in kwargs.items()}
+        realresult = getattr(obj, methodname)(*args, **kwargs)
+        self.assertEqual(
+            result,
+            realresult
+        )
+        # if the original is returned make sure that
+        # this doesn't happen with subclasses
+        if obj is realresult:
+            try:
+                class subtype(self.__class__.type2test):
+                    pass
+            except TypeError:
+                pass  # Skip this if we can't subclass
+            else:
+                obj = subtype(obj)
+                realresult = getattr(obj, methodname)(*args)
+                self.assertIsNot(obj, realresult)
+
+    # check that obj.method(*args) raises exc
+    def checkraises(self, exc, obj, methodname, *args):
+        obj = self.fixtype(obj)
+        args = self.fixtype(args)
+        with self.assertRaises(exc) as cm:
+            getattr(obj, methodname)(*args)
+        self.assertNotEqual(str(cm.exception), '')
+
+    # call obj.method(*args) without any checks
+    def checkcall(self, obj, methodname, *args):
+        obj = self.fixtype(obj)
+        args = self.fixtype(args)
+        getattr(obj, methodname)(*args)
+
+    def test_count(self):
+        self.checkequal(3, 'aaa', 'count', 'a')
+        self.checkequal(0, 'aaa', 'count', 'b')
+        self.checkequal(3, 'aaa', 'count', 'a')
+        self.checkequal(0, 'aaa', 'count', 'b')
+        self.checkequal(3, 'aaa', 'count', 'a')
+        self.checkequal(0, 'aaa', 'count', 'b')
+        self.checkequal(0, 'aaa', 'count', 'b')
+        self.checkequal(2, 'aaa', 'count', 'a', 1)
+        self.checkequal(0, 'aaa', 'count', 'a', 10)
+        self.checkequal(1, 'aaa', 'count', 'a', -1)
+        self.checkequal(3, 'aaa', 'count', 'a', -10)
+        self.checkequal(1, 'aaa', 'count', 'a', 0, 1)
+        self.checkequal(3, 'aaa', 'count', 'a', 0, 10)
+        self.checkequal(2, 'aaa', 'count', 'a', 0, -1)
+        self.checkequal(0, 'aaa', 'count', 'a', 0, -10)
+        self.checkequal(3, 'aaa', 'count', '', 1)
+        self.checkequal(1, 'aaa', 'count', '', 3)
+        self.checkequal(0, 'aaa', 'count', '', 10)
+        self.checkequal(2, 'aaa', 'count', '', -1)
+        self.checkequal(4, 'aaa', 'count', '', -10)
+
+        self.checkequal(1, '', 'count', '')
+        self.checkequal(0, '', 'count', '', 1, 1)
+        self.checkequal(0, '', 'count', '', sys.maxsize, 0)
+
+        self.checkequal(0, '', 'count', 'xx')
+        self.checkequal(0, '', 'count', 'xx', 1, 1)
+        self.checkequal(0, '', 'count', 'xx', sys.maxsize, 0)
+
+        self.checkraises(TypeError, 'hello', 'count')
+
+        if self.contains_bytes:
+            self.checkequal(0, 'hello', 'count', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'count', 42)
+
+        # For a variety of combinations,
+        #    verify that str.count() matches an equivalent function
+        #    replacing all occurrences and then differencing the string lengths
+        charset = ['', 'a', 'b']
+        digits = 7
+        base = len(charset)
+        teststrings = set()
+        for i in range(base ** digits):
+            entry = []
+            for j in range(digits):
+                i, m = divmod(i, base)
+                entry.append(charset[m])
+            teststrings.add(''.join(entry))
+        teststrings = [self.fixtype(ts) for ts in teststrings]
+        for i in teststrings:
+            n = len(i)
+            for j in teststrings:
+                r1 = i.count(j)
+                if j:
+                    r2, rem = divmod(n - len(i.replace(j, self.fixtype(''))),
+                                     len(j))
+                else:
+                    r2, rem = len(i)+1, 0
+                if rem or r1 != r2:
+                    self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i))
+                    self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i))
+
+    def test_find(self):
+        self.checkequal(0, 'abcdefghiabc', 'find', 'abc')
+        self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1)
+        self.checkequal(-1, 'abcdefghiabc', 'find', 'def', 4)
+
+        self.checkequal(0, 'abc', 'find', '', 0)
+        self.checkequal(3, 'abc', 'find', '', 3)
+        self.checkequal(-1, 'abc', 'find', '', 4)
+
+        # to check the ability to pass None as defaults
+        self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a')
+        self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4)
+        self.checkequal(-1, 'rrarrrrrrrrra', 'find', 'a', 4, 6)
+        self.checkequal(12, 'rrarrrrrrrrra', 'find', 'a', 4, None)
+        self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a', None, 6)
+
+        self.checkraises(TypeError, 'hello', 'find')
+
+        if self.contains_bytes:
+            self.checkequal(-1, 'hello', 'find', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'find', 42)
+
+        self.checkequal(0, '', 'find', '')
+        self.checkequal(-1, '', 'find', '', 1, 1)
+        self.checkequal(-1, '', 'find', '', sys.maxsize, 0)
+
+        self.checkequal(-1, '', 'find', 'xx')
+        self.checkequal(-1, '', 'find', 'xx', 1, 1)
+        self.checkequal(-1, '', 'find', 'xx', sys.maxsize, 0)
+
+        # issue 7458
+        self.checkequal(-1, 'ab', 'find', 'xxx', sys.maxsize + 1, 0)
+
+        # For a variety of combinations,
+        #    verify that str.find() matches __contains__
+        #    and that the found substring is really at that location
+        charset = ['', 'a', 'b', 'c']
+        digits = 5
+        base = len(charset)
+        teststrings = set()
+        for i in range(base ** digits):
+            entry = []
+            for j in range(digits):
+                i, m = divmod(i, base)
+                entry.append(charset[m])
+            teststrings.add(''.join(entry))
+        teststrings = [self.fixtype(ts) for ts in teststrings]
+        for i in teststrings:
+            for j in teststrings:
+                loc = i.find(j)
+                r1 = (loc != -1)
+                r2 = j in i
+                self.assertEqual(r1, r2)
+                if loc != -1:
+                    self.assertEqual(i[loc:loc+len(j)], j)
+
+    def test_rfind(self):
+        self.checkequal(9,  'abcdefghiabc', 'rfind', 'abc')
+        self.checkequal(12, 'abcdefghiabc', 'rfind', '')
+        self.checkequal(0, 'abcdefghiabc', 'rfind', 'abcd')
+        self.checkequal(-1, 'abcdefghiabc', 'rfind', 'abcz')
+
+        self.checkequal(3, 'abc', 'rfind', '', 0)
+        self.checkequal(3, 'abc', 'rfind', '', 3)
+        self.checkequal(-1, 'abc', 'rfind', '', 4)
+
+        # to check the ability to pass None as defaults
+        self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a')
+        self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4)
+        self.checkequal(-1, 'rrarrrrrrrrra', 'rfind', 'a', 4, 6)
+        self.checkequal(12, 'rrarrrrrrrrra', 'rfind', 'a', 4, None)
+        self.checkequal( 2, 'rrarrrrrrrrra', 'rfind', 'a', None, 6)
+
+        self.checkraises(TypeError, 'hello', 'rfind')
+
+        if self.contains_bytes:
+            self.checkequal(-1, 'hello', 'rfind', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'rfind', 42)
+
+        # For a variety of combinations,
+        #    verify that str.rfind() matches __contains__
+        #    and that the found substring is really at that location
+        charset = ['', 'a', 'b', 'c']
+        digits = 5
+        base = len(charset)
+        teststrings = set()
+        for i in range(base ** digits):
+            entry = []
+            for j in range(digits):
+                i, m = divmod(i, base)
+                entry.append(charset[m])
+            teststrings.add(''.join(entry))
+        teststrings = [self.fixtype(ts) for ts in teststrings]
+        for i in teststrings:
+            for j in teststrings:
+                loc = i.rfind(j)
+                r1 = (loc != -1)
+                r2 = j in i
+                self.assertEqual(r1, r2)
+                if loc != -1:
+                    self.assertEqual(i[loc:loc+len(j)], j)
+
+        # issue 7458
+        self.checkequal(-1, 'ab', 'rfind', 'xxx', sys.maxsize + 1, 0)
+
+        # issue #15534
+        self.checkequal(0, '<......\u043c...', "rfind", "<")
+
+    def test_index(self):
+        self.checkequal(0, 'abcdefghiabc', 'index', '')
+        self.checkequal(3, 'abcdefghiabc', 'index', 'def')
+        self.checkequal(0, 'abcdefghiabc', 'index', 'abc')
+        self.checkequal(9, 'abcdefghiabc', 'index', 'abc', 1)
+
+        self.checkraises(ValueError, 'abcdefghiabc', 'index', 'hib')
+        self.checkraises(ValueError, 'abcdefghiab', 'index', 'abc', 1)
+        self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', 8)
+        self.checkraises(ValueError, 'abcdefghi', 'index', 'ghi', -1)
+
+        # to check the ability to pass None as defaults
+        self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a')
+        self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4)
+        self.checkraises(ValueError, 'rrarrrrrrrrra', 'index', 'a', 4, 6)
+        self.checkequal(12, 'rrarrrrrrrrra', 'index', 'a', 4, None)
+        self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a', None, 6)
+
+        self.checkraises(TypeError, 'hello', 'index')
+
+        if self.contains_bytes:
+            self.checkraises(ValueError, 'hello', 'index', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'index', 42)
+
+    def test_rindex(self):
+        self.checkequal(12, 'abcdefghiabc', 'rindex', '')
+        self.checkequal(3,  'abcdefghiabc', 'rindex', 'def')
+        self.checkequal(9,  'abcdefghiabc', 'rindex', 'abc')
+        self.checkequal(0,  'abcdefghiabc', 'rindex', 'abc', 0, -1)
+
+        self.checkraises(ValueError, 'abcdefghiabc', 'rindex', 'hib')
+        self.checkraises(ValueError, 'defghiabc', 'rindex', 'def', 1)
+        self.checkraises(ValueError, 'defghiabc', 'rindex', 'abc', 0, -1)
+        self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, 8)
+        self.checkraises(ValueError, 'abcdefghi', 'rindex', 'ghi', 0, -1)
+
+        # to check the ability to pass None as defaults
+        self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a')
+        self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4)
+        self.checkraises(ValueError, 'rrarrrrrrrrra', 'rindex', 'a', 4, 6)
+        self.checkequal(12, 'rrarrrrrrrrra', 'rindex', 'a', 4, None)
+        self.checkequal( 2, 'rrarrrrrrrrra', 'rindex', 'a', None, 6)
+
+        self.checkraises(TypeError, 'hello', 'rindex')
+
+        if self.contains_bytes:
+            self.checkraises(ValueError, 'hello', 'rindex', 42)
+        else:
+            self.checkraises(TypeError, 'hello', 'rindex', 42)
+
+    def test_lower(self):
+        self.checkequal('hello', 'HeLLo', 'lower')
+        self.checkequal('hello', 'hello', 'lower')
+        self.checkraises(TypeError, 'hello', 'lower', 42)
+
+    def test_upper(self):
+        self.checkequal('HELLO', 'HeLLo', 'upper')
+        self.checkequal('HELLO', 'HELLO', 'upper')
+        self.checkraises(TypeError, 'hello', 'upper', 42)
+
+    def test_expandtabs(self):
+        self.checkequal('abc\rab      def\ng       hi', 'abc\rab\tdef\ng\thi',
+                        'expandtabs')
+        self.checkequal('abc\rab      def\ng       hi', 'abc\rab\tdef\ng\thi',
+                        'expandtabs', 8)
+        self.checkequal('abc\rab  def\ng   hi', 'abc\rab\tdef\ng\thi',
+                        'expandtabs', 4)
+        self.checkequal('abc\r\nab      def\ng       hi', 'abc\r\nab\tdef\ng\thi',
+                        'expandtabs')
+        self.checkequal('abc\r\nab      def\ng       hi', 'abc\r\nab\tdef\ng\thi',
+                        'expandtabs', 8)
+        self.checkequal('abc\r\nab  def\ng   hi', 'abc\r\nab\tdef\ng\thi',
+                        'expandtabs', 4)
+        self.checkequal('abc\r\nab\r\ndef\ng\r\nhi', 'abc\r\nab\r\ndef\ng\r\nhi',
+                        'expandtabs', 4)
+        # check keyword args
+        self.checkequal('abc\rab      def\ng       hi', 'abc\rab\tdef\ng\thi',
+                        'expandtabs', tabsize=8)
+        self.checkequal('abc\rab  def\ng   hi', 'abc\rab\tdef\ng\thi',
+                        'expandtabs', tabsize=4)
+
+        self.checkequal('  a\n b', ' \ta\n\tb', 'expandtabs', 1)
+
+        self.checkraises(TypeError, 'hello', 'expandtabs', 42, 42)
+        # This test is only valid when sizeof(int) == sizeof(void*) == 4.
+        if sys.maxsize < (1 << 32) and struct.calcsize('P') == 4:
+            self.checkraises(OverflowError,
+                             '\ta\n\tb', 'expandtabs', sys.maxsize)
+
+    def test_split(self):
+        # by a char
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|')
+        self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0)
+        self.checkequal(['a', 'b|c|d'], 'a|b|c|d', 'split', '|', 1)
+        self.checkequal(['a', 'b', 'c|d'], 'a|b|c|d', 'split', '|', 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|', 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', '|',
+                        sys.maxsize-2)
+        self.checkequal(['a|b|c|d'], 'a|b|c|d', 'split', '|', 0)
+        self.checkequal(['a', '', 'b||c||d'], 'a||b||c||d', 'split', '|', 2)
+        self.checkequal(['abcd'], 'abcd', 'split', '|')
+        self.checkequal([''], '', 'split', '|')
+        self.checkequal(['endcase ', ''], 'endcase |', 'split', '|')
+        self.checkequal(['', ' startcase'], '| startcase', 'split', '|')
+        self.checkequal(['', 'bothcase', ''], '|bothcase|', 'split', '|')
+        self.checkequal(['a', '', 'b\x00c\x00d'], 'a\x00\x00b\x00c\x00d', 'split', '\x00', 2)
+
+        self.checkequal(['a']*20, ('a|'*20)[:-1], 'split', '|')
+        self.checkequal(['a']*15 +['a|a|a|a|a'],
+                                   ('a|'*20)[:-1], 'split', '|', 15)
+
+        # by string
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//')
+        self.checkequal(['a', 'b//c//d'], 'a//b//c//d', 'split', '//', 1)
+        self.checkequal(['a', 'b', 'c//d'], 'a//b//c//d', 'split', '//', 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//', 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'split', '//',
+                        sys.maxsize-10)
+        self.checkequal(['a//b//c//d'], 'a//b//c//d', 'split', '//', 0)
+        self.checkequal(['a', '', 'b////c////d'], 'a////b////c////d', 'split', '//', 2)
+        self.checkequal(['endcase ', ''], 'endcase test', 'split', 'test')
+        self.checkequal(['', ' begincase'], 'test begincase', 'split', 'test')
+        self.checkequal(['', ' bothcase ', ''], 'test bothcase test',
+                        'split', 'test')
+        self.checkequal(['a', 'bc'], 'abbbc', 'split', 'bb')
+        self.checkequal(['', ''], 'aaa', 'split', 'aaa')
+        self.checkequal(['aaa'], 'aaa', 'split', 'aaa', 0)
+        self.checkequal(['ab', 'ab'], 'abbaab', 'split', 'ba')
+        self.checkequal(['aaaa'], 'aaaa', 'split', 'aab')
+        self.checkequal([''], '', 'split', 'aaa')
+        self.checkequal(['aa'], 'aa', 'split', 'aaa')
+        self.checkequal(['A', 'bobb'], 'Abbobbbobb', 'split', 'bbobb')
+        self.checkequal(['A', 'B', ''], 'AbbobbBbbobb', 'split', 'bbobb')
+
+        self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH')
+        self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'split', 'BLAH', 19)
+        self.checkequal(['a']*18 + ['aBLAHa'], ('aBLAH'*20)[:-4],
+                        'split', 'BLAH', 18)
+
+        # with keyword args
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'split', sep='|')
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', '|', maxsplit=1)
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', sep='|', maxsplit=1)
+        self.checkequal(['a', 'b|c|d'],
+                        'a|b|c|d', 'split', maxsplit=1, sep='|')
+        self.checkequal(['a', 'b c d'],
+                        'a b c d', 'split', maxsplit=1)
+
+        # argument type
+        self.checkraises(TypeError, 'hello', 'split', 42, 42, 42)
+
+        # null case
+        self.checkraises(ValueError, 'hello', 'split', '')
+        self.checkraises(ValueError, 'hello', 'split', '', 0)
+
+    def test_rsplit(self):
+        # by a char
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|')
+        self.checkequal(['a|b|c', 'd'], 'a|b|c|d', 'rsplit', '|', 1)
+        self.checkequal(['a|b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|', 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', '|',
+                        sys.maxsize-100)
+        self.checkequal(['a|b|c|d'], 'a|b|c|d', 'rsplit', '|', 0)
+        self.checkequal(['a||b||c', '', 'd'], 'a||b||c||d', 'rsplit', '|', 2)
+        self.checkequal(['abcd'], 'abcd', 'rsplit', '|')
+        self.checkequal([''], '', 'rsplit', '|')
+        self.checkequal(['', ' begincase'], '| begincase', 'rsplit', '|')
+        self.checkequal(['endcase ', ''], 'endcase |', 'rsplit', '|')
+        self.checkequal(['', 'bothcase', ''], '|bothcase|', 'rsplit', '|')
+
+        self.checkequal(['a\x00\x00b', 'c', 'd'], 'a\x00\x00b\x00c\x00d', 'rsplit', '\x00', 2)
+
+        self.checkequal(['a']*20, ('a|'*20)[:-1], 'rsplit', '|')
+        self.checkequal(['a|a|a|a|a']+['a']*15,
+                        ('a|'*20)[:-1], 'rsplit', '|', 15)
+
+        # by string
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//')
+        self.checkequal(['a//b//c', 'd'], 'a//b//c//d', 'rsplit', '//', 1)
+        self.checkequal(['a//b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//', 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a//b//c//d', 'rsplit', '//',
+                        sys.maxsize-5)
+        self.checkequal(['a//b//c//d'], 'a//b//c//d', 'rsplit', '//', 0)
+        self.checkequal(['a////b////c', '', 'd'], 'a////b////c////d', 'rsplit', '//', 2)
+        self.checkequal(['', ' begincase'], 'test begincase', 'rsplit', 'test')
+        self.checkequal(['endcase ', ''], 'endcase test', 'rsplit', 'test')
+        self.checkequal(['', ' bothcase ', ''], 'test bothcase test',
+                        'rsplit', 'test')
+        self.checkequal(['ab', 'c'], 'abbbc', 'rsplit', 'bb')
+        self.checkequal(['', ''], 'aaa', 'rsplit', 'aaa')
+        self.checkequal(['aaa'], 'aaa', 'rsplit', 'aaa', 0)
+        self.checkequal(['ab', 'ab'], 'abbaab', 'rsplit', 'ba')
+        self.checkequal(['aaaa'], 'aaaa', 'rsplit', 'aab')
+        self.checkequal([''], '', 'rsplit', 'aaa')
+        self.checkequal(['aa'], 'aa', 'rsplit', 'aaa')
+        self.checkequal(['bbob', 'A'], 'bbobbbobbA', 'rsplit', 'bbobb')
+        self.checkequal(['', 'B', 'A'], 'bbobbBbbobbA', 'rsplit', 'bbobb')
+
+        self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH')
+        self.checkequal(['a']*20, ('aBLAH'*20)[:-4], 'rsplit', 'BLAH', 19)
+        self.checkequal(['aBLAHa'] + ['a']*18, ('aBLAH'*20)[:-4],
+                        'rsplit', 'BLAH', 18)
+
+        # with keyword args
+        self.checkequal(['a', 'b', 'c', 'd'], 'a|b|c|d', 'rsplit', sep='|')
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', '|', maxsplit=1)
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', sep='|', maxsplit=1)
+        self.checkequal(['a|b|c', 'd'],
+                        'a|b|c|d', 'rsplit', maxsplit=1, sep='|')
+        self.checkequal(['a b c', 'd'],
+                        'a b c d', 'rsplit', maxsplit=1)
+
+        # argument type
+        self.checkraises(TypeError, 'hello', 'rsplit', 42, 42, 42)
+
+        # null case
+        self.checkraises(ValueError, 'hello', 'rsplit', '')
+        self.checkraises(ValueError, 'hello', 'rsplit', '', 0)
+
+    def test_replace(self):
+        EQ = self.checkequal
+
+        # Operations on the empty string
+        EQ("", "", "replace", "", "")
+        EQ("A", "", "replace", "", "A")
+        EQ("", "", "replace", "A", "")
+        EQ("", "", "replace", "A", "A")
+        EQ("", "", "replace", "", "", 100)
+        EQ("", "", "replace", "", "", sys.maxsize)
+
+        # interleave (from=="", 'to' gets inserted everywhere)
+        EQ("A", "A", "replace", "", "")
+        EQ("*A*", "A", "replace", "", "*")
+        EQ("*1A*1", "A", "replace", "", "*1")
+        EQ("*-#A*-#", "A", "replace", "", "*-#")
+        EQ("*-A*-A*-", "AA", "replace", "", "*-")
+        EQ("*-A*-A*-", "AA", "replace", "", "*-", -1)
+        EQ("*-A*-A*-", "AA", "replace", "", "*-", sys.maxsize)
+        EQ("*-A*-A*-", "AA", "replace", "", "*-", 4)
+        EQ("*-A*-A*-", "AA", "replace", "", "*-", 3)
+        EQ("*-A*-A", "AA", "replace", "", "*-", 2)
+        EQ("*-AA", "AA", "replace", "", "*-", 1)
+        EQ("AA", "AA", "replace", "", "*-", 0)
+
+        # single character deletion (from=="A", to=="")
+        EQ("", "A", "replace", "A", "")
+        EQ("", "AAA", "replace", "A", "")
+        EQ("", "AAA", "replace", "A", "", -1)
+        EQ("", "AAA", "replace", "A", "", sys.maxsize)
+        EQ("", "AAA", "replace", "A", "", 4)
+        EQ("", "AAA", "replace", "A", "", 3)
+        EQ("A", "AAA", "replace", "A", "", 2)
+        EQ("AA", "AAA", "replace", "A", "", 1)
+        EQ("AAA", "AAA", "replace", "A", "", 0)
+        EQ("", "AAAAAAAAAA", "replace", "A", "")
+        EQ("BCD", "ABACADA", "replace", "A", "")
+        EQ("BCD", "ABACADA", "replace", "A", "", -1)
+        EQ("BCD", "ABACADA", "replace", "A", "", sys.maxsize)
+        EQ("BCD", "ABACADA", "replace", "A", "", 5)
+        EQ("BCD", "ABACADA", "replace", "A", "", 4)
+        EQ("BCDA", "ABACADA", "replace", "A", "", 3)
+        EQ("BCADA", "ABACADA", "replace", "A", "", 2)
+        EQ("BACADA", "ABACADA", "replace", "A", "", 1)
+        EQ("ABACADA", "ABACADA", "replace", "A", "", 0)
+        EQ("BCD", "ABCAD", "replace", "A", "")
+        EQ("BCD", "ABCADAA", "replace", "A", "")
+        EQ("BCD", "BCD", "replace", "A", "")
+        EQ("*************", "*************", "replace", "A", "")
+        EQ("^A^", "^"+"A"*1000+"^", "replace", "A", "", 999)
+
+        # substring deletion (from=="the", to=="")
+        EQ("", "the", "replace", "the", "")
+        EQ("ater", "theater", "replace", "the", "")
+        EQ("", "thethe", "replace", "the", "")
+        EQ("", "thethethethe", "replace", "the", "")
+        EQ("aaaa", "theatheatheathea", "replace", "the", "")
+        EQ("that", "that", "replace", "the", "")
+        EQ("thaet", "thaet", "replace", "the", "")
+        EQ("here and re", "here and there", "replace", "the", "")
+        EQ("here and re and re", "here and there and there",
+           "replace", "the", "", sys.maxsize)
+        EQ("here and re and re", "here and there and there",
+           "replace", "the", "", -1)
+        EQ("here and re and re", "here and there and there",
+           "replace", "the", "", 3)
+        EQ("here and re and re", "here and there and there",
+           "replace", "the", "", 2)
+        EQ("here and re and there", "here and there and there",
+           "replace", "the", "", 1)
+        EQ("here and there and there", "here and there and there",
+           "replace", "the", "", 0)
+        EQ("here and re and re", "here and there and there", "replace", "the", "")
+
+        EQ("abc", "abc", "replace", "the", "")
+        EQ("abcdefg", "abcdefg", "replace", "the", "")
+
+        # substring deletion (from=="bob", to=="")
+        EQ("bob", "bbobob", "replace", "bob", "")
+        EQ("bobXbob", "bbobobXbbobob", "replace", "bob", "")
+        EQ("aaaaaaa", "aaaaaaabob", "replace", "bob", "")
+        EQ("aaaaaaa", "aaaaaaa", "replace", "bob", "")
+
+        # single character replace in place (len(from)==len(to)==1)
+        EQ("Who goes there?", "Who goes there?", "replace", "o", "o")
+        EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O")
+        EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", sys.maxsize)
+        EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", -1)
+        EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 3)
+        EQ("WhO gOes there?", "Who goes there?", "replace", "o", "O", 2)
+        EQ("WhO goes there?", "Who goes there?", "replace", "o", "O", 1)
+        EQ("Who goes there?", "Who goes there?", "replace", "o", "O", 0)
+
+        EQ("Who goes there?", "Who goes there?", "replace", "a", "q")
+        EQ("who goes there?", "Who goes there?", "replace", "W", "w")
+        EQ("wwho goes there?ww", "WWho goes there?WW", "replace", "W", "w")
+        EQ("Who goes there!", "Who goes there?", "replace", "?", "!")
+        EQ("Who goes there!!", "Who goes there??", "replace", "?", "!")
+
+        EQ("Who goes there?", "Who goes there?", "replace", ".", "!")
+
+        # substring replace in place (len(from)==len(to) > 1)
+        EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**")
+        EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", sys.maxsize)
+        EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", -1)
+        EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 4)
+        EQ("Th** ** a t**sue", "This is a tissue", "replace", "is", "**", 3)
+        EQ("Th** ** a tissue", "This is a tissue", "replace", "is", "**", 2)
+        EQ("Th** is a tissue", "This is a tissue", "replace", "is", "**", 1)
+        EQ("This is a tissue", "This is a tissue", "replace", "is", "**", 0)
+        EQ("cobob", "bobob", "replace", "bob", "cob")
+        EQ("cobobXcobocob", "bobobXbobobob", "replace", "bob", "cob")
+        EQ("bobob", "bobob", "replace", "bot", "bot")
+
+        # replace single character (len(from)==1, len(to)>1)
+        EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK")
+        EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", -1)
+        EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", sys.maxsize)
+        EQ("ReyKKjaviKK", "Reykjavik", "replace", "k", "KK", 2)
+        EQ("ReyKKjavik", "Reykjavik", "replace", "k", "KK", 1)
+        EQ("Reykjavik", "Reykjavik", "replace", "k", "KK", 0)
+        EQ("A----B----C----", "A.B.C.", "replace", ".", "----")
+        # issue #15534
+        EQ('...\u043c......<', '...\u043c......<', "replace", "<", "<")
+
+        EQ("Reykjavik", "Reykjavik", "replace", "q", "KK")
+
+        # replace substring (len(from)>1, len(to)!=len(from))
+        EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam",
+           "replace", "spam", "ham")
+        EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", sys.maxsize)
+        EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", -1)
+        EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", 4)
+        EQ("ham, ham, eggs and ham", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", 3)
+        EQ("ham, ham, eggs and spam", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", 2)
+        EQ("ham, spam, eggs and spam", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", 1)
+        EQ("spam, spam, eggs and spam", "spam, spam, eggs and spam",
+           "replace", "spam", "ham", 0)
+
+        EQ("bobob", "bobobob", "replace", "bobob", "bob")
+        EQ("bobobXbobob", "bobobobXbobobob", "replace", "bobob", "bob")
+        EQ("BOBOBOB", "BOBOBOB", "replace", "bob", "bobby")
+
+        self.checkequal('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1)
+        self.checkequal('onetwothree', 'one!two!three!', 'replace', '!', '')
+        self.checkequal('one@two@three!', 'one!two!three!', 'replace', '!', '@', 2)
+        self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 3)
+        self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@', 4)
+        self.checkequal('one!two!three!', 'one!two!three!', 'replace', '!', '@', 0)
+        self.checkequal('one@two@three@', 'one!two!three!', 'replace', '!', '@')
+        self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@')
+        self.checkequal('one!two!three!', 'one!two!three!', 'replace', 'x', '@', 2)
+        self.checkequal('-a-b-c-', 'abc', 'replace', '', '-')
+        self.checkequal('-a-b-c', 'abc', 'replace', '', '-', 3)
+        self.checkequal('abc', 'abc', 'replace', '', '-', 0)
+        self.checkequal('', '', 'replace', '', '')
+        self.checkequal('abc', 'abc', 'replace', 'ab', '--', 0)
+        self.checkequal('abc', 'abc', 'replace', 'xy', '--')
+        # Next three for SF bug 422088: [OSF1 alpha] string.replace(); died with
+        # MemoryError due to empty result (platform malloc issue when requesting
+        # 0 bytes).
+        self.checkequal('', '123', 'replace', '123', '')
+        self.checkequal('', '123123', 'replace', '123', '')
+        self.checkequal('x', '123x123', 'replace', '123', '')
+
+        self.checkraises(TypeError, 'hello', 'replace')
+        self.checkraises(TypeError, 'hello', 'replace', 42)
+        self.checkraises(TypeError, 'hello', 'replace', 42, 'h')
+        self.checkraises(TypeError, 'hello', 'replace', 'h', 42)
+
+    @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4,
+                     'only applies to 32-bit platforms')
+    def test_replace_overflow(self):
+        # Check for overflow checking on 32 bit machines
+        A2_16 = "A" * (2**16)
+        self.checkraises(OverflowError, A2_16, "replace", "", A2_16)
+        self.checkraises(OverflowError, A2_16, "replace", "A", A2_16)
+        self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16)
+
+    def test_capitalize(self):
+        self.checkequal(' hello ', ' hello ', 'capitalize')
+        self.checkequal('Hello ', 'Hello ','capitalize')
+        self.checkequal('Hello ', 'hello ','capitalize')
+        self.checkequal('Aaaa', 'aaaa', 'capitalize')
+        self.checkequal('Aaaa', 'AaAa', 'capitalize')
+
+        self.checkraises(TypeError, 'hello', 'capitalize', 42)
+
+    def test_additional_split(self):
+        self.checkequal(['this', 'is', 'the', 'split', 'function'],
+            'this is the split function', 'split')
+
+        # by whitespace
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'split')
+        self.checkequal(['a', 'b c d'], 'a b c d', 'split', None, 1)
+        self.checkequal(['a', 'b', 'c d'], 'a b c d', 'split', None, 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None, 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'split', None,
+                        sys.maxsize-1)
+        self.checkequal(['a b c d'], 'a b c d', 'split', None, 0)
+        self.checkequal(['a b c d'], '  a b c d', 'split', None, 0)
+        self.checkequal(['a', 'b', 'c  d'], 'a  b  c  d', 'split', None, 2)
+
+        self.checkequal([], '         ', 'split')
+        self.checkequal(['a'], '  a    ', 'split')
+        self.checkequal(['a', 'b'], '  a    b   ', 'split')
+        self.checkequal(['a', 'b   '], '  a    b   ', 'split', None, 1)
+        self.checkequal(['a    b   c   '], '  a    b   c   ', 'split', None, 0)
+        self.checkequal(['a', 'b   c   '], '  a    b   c   ', 'split', None, 1)
+        self.checkequal(['a', 'b', 'c   '], '  a    b   c   ', 'split', None, 2)
+        self.checkequal(['a', 'b', 'c'], '  a    b   c   ', 'split', None, 3)
+        self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'split')
+        aaa = ' a '*20
+        self.checkequal(['a']*20, aaa, 'split')
+        self.checkequal(['a'] + [aaa[4:]], aaa, 'split', None, 1)
+        self.checkequal(['a']*19 + ['a '], aaa, 'split', None, 19)
+
+        for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf',
+                  'arf\fbarf', 'arf\vbarf'):
+            self.checkequal(['arf', 'barf'], b, 'split')
+            self.checkequal(['arf', 'barf'], b, 'split', None)
+            self.checkequal(['arf', 'barf'], b, 'split', None, 2)
+
+    def test_additional_rsplit(self):
+        self.checkequal(['this', 'is', 'the', 'rsplit', 'function'],
+                         'this is the rsplit function', 'rsplit')
+
+        # by whitespace
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d ', 'rsplit')
+        self.checkequal(['a b c', 'd'], 'a b c d', 'rsplit', None, 1)
+        self.checkequal(['a b', 'c', 'd'], 'a b c d', 'rsplit', None, 2)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 3)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None, 4)
+        self.checkequal(['a', 'b', 'c', 'd'], 'a b c d', 'rsplit', None,
+                        sys.maxsize-20)
+        self.checkequal(['a b c d'], 'a b c d', 'rsplit', None, 0)
+        self.checkequal(['a b c d'], 'a b c d  ', 'rsplit', None, 0)
+        self.checkequal(['a  b', 'c', 'd'], 'a  b  c  d', 'rsplit', None, 2)
+
+        self.checkequal([], '         ', 'rsplit')
+        self.checkequal(['a'], '  a    ', 'rsplit')
+        self.checkequal(['a', 'b'], '  a    b   ', 'rsplit')
+        self.checkequal(['  a', 'b'], '  a    b   ', 'rsplit', None, 1)
+        self.checkequal(['  a    b   c'], '  a    b   c   ', 'rsplit',
+                        None, 0)
+        self.checkequal(['  a    b','c'], '  a    b   c   ', 'rsplit',
+                        None, 1)
+        self.checkequal(['  a', 'b', 'c'], '  a    b   c   ', 'rsplit',
+                        None, 2)
+        self.checkequal(['a', 'b', 'c'], '  a    b   c   ', 'rsplit',
+                        None, 3)
+        self.checkequal(['a', 'b'], '\n\ta \t\r b \v ', 'rsplit', None, 88)
+        aaa = ' a '*20
+        self.checkequal(['a']*20, aaa, 'rsplit')
+        self.checkequal([aaa[:-4]] + ['a'], aaa, 'rsplit', None, 1)
+        self.checkequal([' a  a'] + ['a']*18, aaa, 'rsplit', None, 18)
+
+        for b in ('arf\tbarf', 'arf\nbarf', 'arf\rbarf',
+                  'arf\fbarf', 'arf\vbarf'):
+            self.checkequal(['arf', 'barf'], b, 'rsplit')
+            self.checkequal(['arf', 'barf'], b, 'rsplit', None)
+            self.checkequal(['arf', 'barf'], b, 'rsplit', None, 2)
+
+    def test_strip_whitespace(self):
+        self.checkequal('hello', '   hello   ', 'strip')
+        self.checkequal('hello   ', '   hello   ', 'lstrip')
+        self.checkequal('   hello', '   hello   ', 'rstrip')
+        self.checkequal('hello', 'hello', 'strip')
+
+        b = ' \t\n\r\f\vabc \t\n\r\f\v'
+        self.checkequal('abc', b, 'strip')
+        self.checkequal('abc \t\n\r\f\v', b, 'lstrip')
+        self.checkequal(' \t\n\r\f\vabc', b, 'rstrip')
+
+        # strip/lstrip/rstrip with None arg
+        self.checkequal('hello', '   hello   ', 'strip', None)
+        self.checkequal('hello   ', '   hello   ', 'lstrip', None)
+        self.checkequal('   hello', '   hello   ', 'rstrip', None)
+        self.checkequal('hello', 'hello', 'strip', None)
+
+    def test_strip(self):
+        # strip/lstrip/rstrip with str arg
+        self.checkequal('hello', 'xyzzyhelloxyzzy', 'strip', 'xyz')
+        self.checkequal('helloxyzzy', 'xyzzyhelloxyzzy', 'lstrip', 'xyz')
+        self.checkequal('xyzzyhello', 'xyzzyhelloxyzzy', 'rstrip', 'xyz')
+        self.checkequal('hello', 'hello', 'strip', 'xyz')
+        self.checkequal('', 'mississippi', 'strip', 'mississippi')
+
+        # only trim the start and end; does not strip internal characters
+        self.checkequal('mississipp', 'mississippi', 'strip', 'i')
+
+        self.checkraises(TypeError, 'hello', 'strip', 42, 42)
+        self.checkraises(TypeError, 'hello', 'lstrip', 42, 42)
+        self.checkraises(TypeError, 'hello', 'rstrip', 42, 42)
+
+    def test_ljust(self):
+        self.checkequal('abc       ', 'abc', 'ljust', 10)
+        self.checkequal('abc   ', 'abc', 'ljust', 6)
+        self.checkequal('abc', 'abc', 'ljust', 3)
+        self.checkequal('abc', 'abc', 'ljust', 2)
+        self.checkequal('abc*******', 'abc', 'ljust', 10, '*')
+        self.checkraises(TypeError, 'abc', 'ljust')
+
+    def test_rjust(self):
+        self.checkequal('       abc', 'abc', 'rjust', 10)
+        self.checkequal('   abc', 'abc', 'rjust', 6)
+        self.checkequal('abc', 'abc', 'rjust', 3)
+        self.checkequal('abc', 'abc', 'rjust', 2)
+        self.checkequal('*******abc', 'abc', 'rjust', 10, '*')
+        self.checkraises(TypeError, 'abc', 'rjust')
+
+    def test_center(self):
+        self.checkequal('   abc    ', 'abc', 'center', 10)
+        self.checkequal(' abc  ', 'abc', 'center', 6)
+        self.checkequal('abc', 'abc', 'center', 3)
+        self.checkequal('abc', 'abc', 'center', 2)
+        self.checkequal('***abc****', 'abc', 'center', 10, '*')
+        self.checkraises(TypeError, 'abc', 'center')
+
+    def test_swapcase(self):
+        self.checkequal('hEllO CoMPuTErS', 'HeLLo cOmpUteRs', 'swapcase')
+
+        self.checkraises(TypeError, 'hello', 'swapcase', 42)
+
+    def test_zfill(self):
+        self.checkequal('123', '123', 'zfill', 2)
+        self.checkequal('123', '123', 'zfill', 3)
+        self.checkequal('0123', '123', 'zfill', 4)
+        self.checkequal('+123', '+123', 'zfill', 3)
+        self.checkequal('+123', '+123', 'zfill', 4)
+        self.checkequal('+0123', '+123', 'zfill', 5)
+        self.checkequal('-123', '-123', 'zfill', 3)
+        self.checkequal('-123', '-123', 'zfill', 4)
+        self.checkequal('-0123', '-123', 'zfill', 5)
+        self.checkequal('000', '', 'zfill', 3)
+        self.checkequal('34', '34', 'zfill', 1)
+        self.checkequal('0034', '34', 'zfill', 4)
+
+        self.checkraises(TypeError, '123', 'zfill')
+
+    def test_islower(self):
+        self.checkequal(False, '', 'islower')
+        self.checkequal(True, 'a', 'islower')
+        self.checkequal(False, 'A', 'islower')
+        self.checkequal(False, '\n', 'islower')
+        self.checkequal(True, 'abc', 'islower')
+        self.checkequal(False, 'aBc', 'islower')
+        self.checkequal(True, 'abc\n', 'islower')
+        self.checkraises(TypeError, 'abc', 'islower', 42)
+
+    def test_isupper(self):
+        self.checkequal(False, '', 'isupper')
+        self.checkequal(False, 'a', 'isupper')
+        self.checkequal(True, 'A', 'isupper')
+        self.checkequal(False, '\n', 'isupper')
+        self.checkequal(True, 'ABC', 'isupper')
+        self.checkequal(False, 'AbC', 'isupper')
+        self.checkequal(True, 'ABC\n', 'isupper')
+        self.checkraises(TypeError, 'abc', 'isupper', 42)
+
+    def test_istitle(self):
+        self.checkequal(False, '', 'istitle')
+        self.checkequal(False, 'a', 'istitle')
+        self.checkequal(True, 'A', 'istitle')
+        self.checkequal(False, '\n', 'istitle')
+        self.checkequal(True, 'A Titlecased Line', 'istitle')
+        self.checkequal(True, 'A\nTitlecased Line', 'istitle')
+        self.checkequal(True, 'A Titlecased, Line', 'istitle')
+        self.checkequal(False, 'Not a capitalized String', 'istitle')
+        self.checkequal(False, 'Not\ta Titlecase String', 'istitle')
+        self.checkequal(False, 'Not--a Titlecase String', 'istitle')
+        self.checkequal(False, 'NOT', 'istitle')
+        self.checkraises(TypeError, 'abc', 'istitle', 42)
+
+    def test_isspace(self):
+        self.checkequal(False, '', 'isspace')
+        self.checkequal(False, 'a', 'isspace')
+        self.checkequal(True, ' ', 'isspace')
+        self.checkequal(True, '\t', 'isspace')
+        self.checkequal(True, '\r', 'isspace')
+        self.checkequal(True, '\n', 'isspace')
+        self.checkequal(True, ' \t\r\n', 'isspace')
+        self.checkequal(False, ' \t\r\na', 'isspace')
+        self.checkraises(TypeError, 'abc', 'isspace', 42)
+
+    def test_isalpha(self):
+        self.checkequal(False, '', 'isalpha')
+        self.checkequal(True, 'a', 'isalpha')
+        self.checkequal(True, 'A', 'isalpha')
+        self.checkequal(False, '\n', 'isalpha')
+        self.checkequal(True, 'abc', 'isalpha')
+        self.checkequal(False, 'aBc123', 'isalpha')
+        self.checkequal(False, 'abc\n', 'isalpha')
+        self.checkraises(TypeError, 'abc', 'isalpha', 42)
+
+    def test_isalnum(self):
+        self.checkequal(False, '', 'isalnum')
+        self.checkequal(True, 'a', 'isalnum')
+        self.checkequal(True, 'A', 'isalnum')
+        self.checkequal(False, '\n', 'isalnum')
+        self.checkequal(True, '123abc456', 'isalnum')
+        self.checkequal(True, 'a1b3c', 'isalnum')
+        self.checkequal(False, 'aBc000 ', 'isalnum')
+        self.checkequal(False, 'abc\n', 'isalnum')
+        self.checkraises(TypeError, 'abc', 'isalnum', 42)
+
+    def test_isascii(self):
+        self.checkequal(True, '', 'isascii')
+        self.checkequal(True, '\x00', 'isascii')
+        self.checkequal(True, '\x7f', 'isascii')
+        self.checkequal(True, '\x00\x7f', 'isascii')
+        self.checkequal(False, '\x80', 'isascii')
+        self.checkequal(False, '\xe9', 'isascii')
+        # bytes.isascii() and bytearray.isascii() has optimization which
+        # check 4 or 8 bytes at once.  So check some alignments.
+        for p in range(8):
+            self.checkequal(True, ' '*p + '\x7f', 'isascii')
+            self.checkequal(False, ' '*p + '\x80', 'isascii')
+            self.checkequal(True, ' '*p + '\x7f' + ' '*8, 'isascii')
+            self.checkequal(False, ' '*p + '\x80' + ' '*8, 'isascii')
+
+    def test_isdigit(self):
+        self.checkequal(False, '', 'isdigit')
+        self.checkequal(False, 'a', 'isdigit')
+        self.checkequal(True, '0', 'isdigit')
+        self.checkequal(True, '0123456789', 'isdigit')
+        self.checkequal(False, '0123456789a', 'isdigit')
+
+        self.checkraises(TypeError, 'abc', 'isdigit', 42)
+
+    def test_title(self):
+        self.checkequal(' Hello ', ' hello ', 'title')
+        self.checkequal('Hello ', 'hello ', 'title')
+        self.checkequal('Hello ', 'Hello ', 'title')
+        self.checkequal('Format This As Title String', "fOrMaT thIs aS titLe String", 'title')
+        self.checkequal('Format,This-As*Title;String', "fOrMaT,thIs-aS*titLe;String", 'title', )
+        self.checkequal('Getint', "getInt", 'title')
+        self.checkraises(TypeError, 'hello', 'title', 42)
+
+    def test_splitlines(self):
+        self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\rghi", 'splitlines')
+        self.checkequal(['abc', 'def', '', 'ghi'], "abc\ndef\n\r\nghi", 'splitlines')
+        self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi", 'splitlines')
+        self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi\n", 'splitlines')
+        self.checkequal(['abc', 'def', 'ghi', ''], "abc\ndef\r\nghi\n\r", 'splitlines')
+        self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", 'splitlines')
+        self.checkequal(['', 'abc', 'def', 'ghi', ''],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', False)
+        self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', True)
+        self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r",
+                        'splitlines', keepends=False)
+        self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'],
+                        "\nabc\ndef\r\nghi\n\r", 'splitlines', keepends=True)
+
+        self.checkraises(TypeError, 'abc', 'splitlines', 42, 42)
+
+
+class CommonTest(BaseTest):
+    # This testcase contains tests that can be used in all
+    # stringlike classes. Currently this is str and UserString.
+
+    def test_hash(self):
+        # SF bug 1054139:  += optimization was not invalidating cached hash value
+        a = self.type2test('DNSSEC')
+        b = self.type2test('')
+        for c in a:
+            b += c
+            hash(b)
+        self.assertEqual(hash(a), hash(b))
+
+    def test_capitalize_nonascii(self):
+        # check that titlecased chars are lowered correctly
+        # \u1ffc is the titlecased char
+        # Note: differs between Py<3.8 and later.
+        #self.checkequal('\u03a9\u0399\u1ff3\u1ff3\u1ff3',
+        #                '\u1ff3\u1ff3\u1ffc\u1ffc', 'capitalize')
+        # check with cased non-letter chars
+        self.checkequal('\u24c5\u24e8\u24e3\u24d7\u24de\u24dd',
+                        '\u24c5\u24ce\u24c9\u24bd\u24c4\u24c3', 'capitalize')
+        self.checkequal('\u24c5\u24e8\u24e3\u24d7\u24de\u24dd',
+                        '\u24df\u24e8\u24e3\u24d7\u24de\u24dd', 'capitalize')
+        self.checkequal('\u2160\u2171\u2172',
+                        '\u2160\u2161\u2162', 'capitalize')
+        self.checkequal('\u2160\u2171\u2172',
+                        '\u2170\u2171\u2172', 'capitalize')
+        # check with Ll chars with no upper - nothing changes here
+        self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7',
+                        '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize')
+
+    def test_list_concat(self):
+        # https://github.com/cython/cython/issues/3426
+        y = []
+        y += 'ab'
+        self.assertEqual('a', y[0])
+        self.assertEqual('b', y[1])
+        self.assertEqual(['a', 'b'], y)
+
+
+class MixinStrUnicodeUserStringTest:
+    # additional tests that only work for
+    # stringlike objects, i.e. str, UserString
+
+    @unittest.skipIf(sys.version_info < (3, 5), 'Python str.startswith() test requires Py3.5+')
+    def test_startswith(self):
+        self.checkequal(True, 'hello', 'startswith', 'he')
+        self.checkequal(True, 'hello', 'startswith', 'hello')
+        self.checkequal(False, 'hello', 'startswith', 'hello world')
+        self.checkequal(True, 'hello', 'startswith', '')
+        self.checkequal(False, 'hello', 'startswith', 'ello')
+        self.checkequal(True, 'hello', 'startswith', 'ello', 1)
+        self.checkequal(True, 'hello', 'startswith', 'o', 4)
+        self.checkequal(False, 'hello', 'startswith', 'o', 5)
+        self.checkequal(True, 'hello', 'startswith', '', 5)
+        self.checkequal(False, 'hello', 'startswith', 'lo', 6)
+        self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3)
+        self.checkequal(True, 'helloworld', 'startswith', 'lowo', 3, 7)
+        self.checkequal(False, 'helloworld', 'startswith', 'lowo', 3, 6)
+        self.checkequal(True, '', 'startswith', '', 0, 1)
+        self.checkequal(True, '', 'startswith', '', 0, 0)
+        self.checkequal(False, '', 'startswith', '', 1, 0)
+
+        # test negative indices
+        self.checkequal(True, 'hello', 'startswith', 'he', 0, -1)
+        self.checkequal(True, 'hello', 'startswith', 'he', -53, -1)
+        self.checkequal(False, 'hello', 'startswith', 'hello', 0, -1)
+        self.checkequal(False, 'hello', 'startswith', 'hello world', -1, -10)
+        self.checkequal(False, 'hello', 'startswith', 'ello', -5)
+        self.checkequal(True, 'hello', 'startswith', 'ello', -4)
+        self.checkequal(False, 'hello', 'startswith', 'o', -2)
+        self.checkequal(True, 'hello', 'startswith', 'o', -1)
+        self.checkequal(True, 'hello', 'startswith', '', -3, -3)
+        self.checkequal(False, 'hello', 'startswith', 'lo', -9)
+
+        self.checkraises(TypeError, 'hello', 'startswith')
+        self.checkraises(TypeError, 'hello', 'startswith', 42)
+
+        # test tuple arguments
+        self.checkequal(True, 'hello', 'startswith', ('he', 'ha'))
+        self.checkequal(False, 'hello', 'startswith', ('lo', 'llo'))
+        self.checkequal(True, 'hello', 'startswith', ('hellox', 'hello'))
+        self.checkequal(False, 'hello', 'startswith', ())
+        self.checkequal(True, 'helloworld', 'startswith', ('hellowo',
+                                                           'rld', 'lowo'), 3)
+        self.checkequal(False, 'helloworld', 'startswith', ('hellowo', 'ello',
+                                                            'rld'), 3)
+        self.checkequal(True, 'hello', 'startswith', ('lo', 'he'), 0, -1)
+        self.checkequal(False, 'hello', 'startswith', ('he', 'hel'), 0, 1)
+        self.checkequal(True, 'hello', 'startswith', ('he', 'hel'), 0, 2)
+
+        self.checkraises(TypeError, 'hello', 'startswith', (42,))
+
+    @unittest.skipIf(sys.version_info < (3, 5), 'Python str.endswith() test requires Py3.5+')
+    def test_endswith(self):
+        self.checkequal(True, 'hello', 'endswith', 'lo')
+        self.checkequal(False, 'hello', 'endswith', 'he')
+        self.checkequal(True, 'hello', 'endswith', '')
+        self.checkequal(False, 'hello', 'endswith', 'hello world')
+        self.checkequal(False, 'helloworld', 'endswith', 'worl')
+        self.checkequal(True, 'helloworld', 'endswith', 'worl', 3, 9)
+        self.checkequal(True, 'helloworld', 'endswith', 'world', 3, 12)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', 1, 7)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', 2, 7)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', 3, 7)
+        self.checkequal(False, 'helloworld', 'endswith', 'lowo', 4, 7)
+        self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, 8)
+        self.checkequal(False, 'ab', 'endswith', 'ab', 0, 1)
+        self.checkequal(False, 'ab', 'endswith', 'ab', 0, 0)
+        self.checkequal(True, '', 'endswith', '', 0, 1)
+        self.checkequal(True, '', 'endswith', '', 0, 0)
+        self.checkequal(False, '', 'endswith', '', 1, 0)
+
+        # test negative indices
+        self.checkequal(True, 'hello', 'endswith', 'lo', -2)
+        self.checkequal(False, 'hello', 'endswith', 'he', -2)
+        self.checkequal(True, 'hello', 'endswith', '', -3, -3)
+        self.checkequal(False, 'hello', 'endswith', 'hello world', -10, -2)
+        self.checkequal(False, 'helloworld', 'endswith', 'worl', -6)
+        self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, -1)
+        self.checkequal(True, 'helloworld', 'endswith', 'worl', -5, 9)
+        self.checkequal(True, 'helloworld', 'endswith', 'world', -7, 12)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', -99, -3)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', -8, -3)
+        self.checkequal(True, 'helloworld', 'endswith', 'lowo', -7, -3)
+        self.checkequal(False, 'helloworld', 'endswith', 'lowo', 3, -4)
+        self.checkequal(False, 'helloworld', 'endswith', 'lowo', -8, -2)
+
+        self.checkraises(TypeError, 'hello', 'endswith')
+        self.checkraises(TypeError, 'hello', 'endswith', 42)
+
+        # test tuple arguments
+        self.checkequal(False, 'hello', 'endswith', ('he', 'ha'))
+        self.checkequal(True, 'hello', 'endswith', ('lo', 'llo'))
+        self.checkequal(True, 'hello', 'endswith', ('hellox', 'hello'))
+        self.checkequal(False, 'hello', 'endswith', ())
+        self.checkequal(True, 'helloworld', 'endswith', ('hellowo',
+                                                           'rld', 'lowo'), 3)
+        self.checkequal(False, 'helloworld', 'endswith', ('hellowo', 'ello',
+                                                            'rld'), 3, -1)
+        self.checkequal(True, 'hello', 'endswith', ('hell', 'ell'), 0, -1)
+        self.checkequal(False, 'hello', 'endswith', ('he', 'hel'), 0, 1)
+        self.checkequal(True, 'hello', 'endswith', ('he', 'hell'), 0, 4)
+
+        self.checkraises(TypeError, 'hello', 'endswith', (42,))
+
+    def test___contains__(self):
+        self.checkequal(True, '', '__contains__', '')
+        self.checkequal(True, 'abc', '__contains__', '')
+        self.checkequal(False, 'abc', '__contains__', '\0')
+        self.checkequal(True, '\0abc', '__contains__', '\0')
+        self.checkequal(True, 'abc\0', '__contains__', '\0')
+        self.checkequal(True, '\0abc', '__contains__', 'a')
+        self.checkequal(True, 'asdf', '__contains__', 'asdf')
+        self.checkequal(False, 'asd', '__contains__', 'asdf')
+        self.checkequal(False, '', '__contains__', 'asdf')
+
+    def test_subscript(self):
+        self.checkequal('a', 'abc', '__getitem__', 0)
+        self.checkequal('c', 'abc', '__getitem__', -1)
+        self.checkequal('a', 'abc', '__getitem__', 0)
+        self.checkequal('abc', 'abc', '__getitem__', slice(0, 3))
+        self.checkequal('abc', 'abc', '__getitem__', slice(0, 1000))
+        self.checkequal('a', 'abc', '__getitem__', slice(0, 1))
+        self.checkequal('', 'abc', '__getitem__', slice(0, 0))
+
+        self.checkraises(TypeError, 'abc', '__getitem__', 'def')
+
+    def test_slice(self):
+        self.checkequal('abc', 'abc', '__getitem__', slice(0, 1000))
+        self.checkequal('abc', 'abc', '__getitem__', slice(0, 3))
+        self.checkequal('ab', 'abc', '__getitem__', slice(0, 2))
+        self.checkequal('bc', 'abc', '__getitem__', slice(1, 3))
+        self.checkequal('b', 'abc', '__getitem__', slice(1, 2))
+        self.checkequal('', 'abc', '__getitem__', slice(2, 2))
+        self.checkequal('', 'abc', '__getitem__', slice(1000, 1000))
+        self.checkequal('', 'abc', '__getitem__', slice(2000, 1000))
+        self.checkequal('', 'abc', '__getitem__', slice(2, 1))
+
+        self.checkraises(TypeError, 'abc', '__getitem__', 'def')
+
+    def test_extended_getslice(self):
+        # Test extended slicing by comparing with list slicing.
+        s = string.ascii_letters + string.digits
+        indices = (0, None, 1, 3, 41, -1, -2, -37)
+        for start in indices:
+            for stop in indices:
+                # Skip step 0 (invalid)
+                for step in indices[1:]:
+                    L = list(s)[start:stop:step]
+                    self.checkequal("".join(L), s, '__getitem__',
+                                    slice(start, stop, step))
+
+    def test_mul(self):
+        self.checkequal('', 'abc', '__mul__', -1)
+        self.checkequal('', 'abc', '__mul__', 0)
+        self.checkequal('abc', 'abc', '__mul__', 1)
+        self.checkequal('abcabcabc', 'abc', '__mul__', 3)
+        self.checkraises(TypeError, 'abc', '__mul__')
+        self.checkraises(TypeError, 'abc', '__mul__', '')
+        # XXX: on a 64-bit system, this doesn't raise an overflow error,
+        # but either raises a MemoryError, or succeeds (if you have 54TiB)
+        #self.checkraises(OverflowError, 10000*'abc', '__mul__', 2000000000)
+
+    def test_join(self):
+        # join now works with any sequence type
+        # moved here, because the argument order is
+        # different in string.join
+        self.checkequal('a b c d', ' ', 'join', ['a', 'b', 'c', 'd'])
+        self.checkequal('abcd', '', 'join', ('a', 'b', 'c', 'd'))
+        self.checkequal('bd', '', 'join', ('', 'b', '', 'd'))
+        self.checkequal('ac', '', 'join', ('a', '', 'c', ''))
+        self.checkequal('w x y z', ' ', 'join', Sequence())
+        self.checkequal('abc', 'a', 'join', ('abc',))
+        #self.checkequal('z', 'a', 'join', UserList(['z']))
+        self.checkequal('a.b.c', '.', 'join', ['a', 'b', 'c'])
+        self.assertRaises(TypeError, '.'.join, ['a', 'b', 3])
+        for i in [5, 25, 125]:
+            self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join',
+                 ['a' * i] * i)
+            self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join',
+                 ('a' * i,) * i)
+
+        #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1())
+        self.checkequal('a b c', ' ', 'join', BadSeq2())
+
+        self.checkraises(TypeError, ' ', 'join')
+        self.checkraises(TypeError, ' ', 'join', None)
+        self.checkraises(TypeError, ' ', 'join', 7)
+        self.checkraises(TypeError, ' ', 'join', [1, 2, bytes()])
+        try:
+            def f():
+                yield 4 + ""
+            self.fixtype(' ').join(f())
+        except TypeError as e:
+            if '+' not in str(e):
+                self.fail('join() ate exception message')
+        else:
+            self.fail('exception not raised')
+
+    def test_formatting(self):
+        self.checkequal('+hello+', '+%s+', '__mod__', 'hello')
+        self.checkequal('+10+', '+%d+', '__mod__', 10)
+        self.checkequal('a', "%c", '__mod__', "a")
+        self.checkequal('a', "%c", '__mod__', "a")
+        self.checkequal('"', "%c", '__mod__', 34)
+        self.checkequal('$', "%c", '__mod__', 36)
+        self.checkequal('10', "%d", '__mod__', 10)
+        self.checkequal('\x7f', "%c", '__mod__', 0x7f)
+
+        for ordinal in (-100, 0x200000):
+            # unicode raises ValueError, str raises OverflowError
+            self.checkraises((ValueError, OverflowError), '%c', '__mod__', ordinal)
+
+        longvalue = sys.maxsize + 10
+        slongvalue = str(longvalue)
+        self.checkequal(' 42', '%3ld', '__mod__', 42)
+        self.checkequal('42', '%d', '__mod__', 42.0)
+        self.checkequal(slongvalue, '%d', '__mod__', longvalue)
+        self.checkcall('%d', '__mod__', float(longvalue))
+        self.checkequal('0042.00', '%07.2f', '__mod__', 42)
+        self.checkequal('0042.00', '%07.2F', '__mod__', 42)
+
+        self.checkraises(TypeError, 'abc', '__mod__')
+        self.checkraises(TypeError, '%(foo)s', '__mod__', 42)
+        self.checkraises(TypeError, '%s%s', '__mod__', (42,))
+        self.checkraises(TypeError, '%c', '__mod__', (None,))
+        self.checkraises(ValueError, '%(foo', '__mod__', {})
+        self.checkraises(TypeError, '%(foo)s %(bar)s', '__mod__', ('foo', 42))
+        self.checkraises(TypeError, '%d', '__mod__', "42") # not numeric
+        self.checkraises(TypeError, '%d', '__mod__', (42+0j)) # no int conversion provided
+
+        # argument names with properly nested brackets are supported
+        self.checkequal('bar', '%((foo))s', '__mod__', {'(foo)': 'bar'})
+
+        # 100 is a magic number in PyUnicode_Format, this forces a resize
+        self.checkequal(103*'a'+'x', '%sx', '__mod__', 103*'a')
+
+        self.checkraises(TypeError, '%*s', '__mod__', ('foo', 'bar'))
+        self.checkraises(TypeError, '%10.*f', '__mod__', ('foo', 42.))
+        self.checkraises(ValueError, '%10', '__mod__', (42,))
+
+        # Outrageously large width or precision should raise ValueError.
+        self.checkraises(ValueError, '%%%df' % (2**64), '__mod__', (3.2))
+        self.checkraises(ValueError, '%%.%df' % (2**64), '__mod__', (3.2))
+        self.checkraises(OverflowError, '%*s', '__mod__',
+                         (sys.maxsize + 1, ''))
+        self.checkraises(OverflowError, '%.*f', '__mod__',
+                         (sys.maxsize + 1, 1. / 7))
+
+        class X(object): pass
+        self.checkraises(TypeError, 'abc', '__mod__', X())
+
+    @support.cpython_only
+    def test_formatting_c_limits(self):
+        from _testcapi import PY_SSIZE_T_MAX, INT_MAX, UINT_MAX
+        SIZE_MAX = (1 << (PY_SSIZE_T_MAX.bit_length() + 1)) - 1
+        self.checkraises(OverflowError, '%*s', '__mod__',
+                         (PY_SSIZE_T_MAX + 1, ''))
+        self.checkraises(OverflowError, '%.*f', '__mod__',
+                         (INT_MAX + 1, 1. / 7))
+        # Issue 15989
+        self.checkraises(OverflowError, '%*s', '__mod__',
+                         (SIZE_MAX + 1, ''))
+        self.checkraises(OverflowError, '%.*f', '__mod__',
+                         (UINT_MAX + 1, 1. / 7))
+
+    def test_floatformatting(self):
+        # float formatting
+        for prec in range(100):
+            format = '%%.%if' % prec
+            value = 0.01
+            for x in range(60):
+                value = value * 3.14159265359 / 3.0 * 10.0
+                self.checkcall(format, "__mod__", value)
+
+    def test_inplace_rewrites(self):
+        # Check that strings don't copy and modify cached single-character strings
+        self.checkequal('a', 'A', 'lower')
+        self.checkequal(True, 'A', 'isupper')
+        self.checkequal('A', 'a', 'upper')
+        self.checkequal(True, 'a', 'islower')
+
+        self.checkequal('a', 'A', 'replace', 'A', 'a')
+        self.checkequal(True, 'A', 'isupper')
+
+        self.checkequal('A', 'a', 'capitalize')
+        self.checkequal(True, 'a', 'islower')
+
+        self.checkequal('A', 'a', 'swapcase')
+        self.checkequal(True, 'a', 'islower')
+
+        self.checkequal('A', 'a', 'title')
+        self.checkequal(True, 'a', 'islower')
+
+    @unittest.skipIf(sys.version_info < (3, 5), 'Python str.partition() test requires Py3.5+')
+    def test_partition(self):
+
+        self.checkequal(('this is the par', 'ti', 'tion method'),
+            'this is the partition method', 'partition', 'ti')
+
+        # from raymond's original specification
+        S = 'http://www.python.org'
+        self.checkequal(('http', '://', 'www.python.org'), S, 'partition', '://')
+        self.checkequal(('http://www.python.org', '', ''), S, 'partition', '?')
+        self.checkequal(('', 'http://', 'www.python.org'), S, 'partition', 'http://')
+        self.checkequal(('http://www.python.', 'org', ''), S, 'partition', 'org')
+
+        self.checkraises(ValueError, S, 'partition', '')
+        self.checkraises(TypeError, S, 'partition', None)
+
+    @unittest.skipIf(sys.version_info < (3, 5), 'Python str.rpartition() test requires Py3.5+')
+    def test_rpartition(self):
+
+        self.checkequal(('this is the rparti', 'ti', 'on method'),
+            'this is the rpartition method', 'rpartition', 'ti')
+
+        # from raymond's original specification
+        S = 'http://www.python.org'
+        self.checkequal(('http', '://', 'www.python.org'), S, 'rpartition', '://')
+        self.checkequal(('', '', 'http://www.python.org'), S, 'rpartition', '?')
+        self.checkequal(('', 'http://', 'www.python.org'), S, 'rpartition', 'http://')
+        self.checkequal(('http://www.python.', 'org', ''), S, 'rpartition', 'org')
+
+        self.checkraises(ValueError, S, 'rpartition', '')
+        self.checkraises(TypeError, S, 'rpartition', None)
+
+    def test_none_arguments(self):
+        # issue 11828
+        s = 'hello'
+        self.checkequal(2, s, 'find', 'l', None)
+        self.checkequal(3, s, 'find', 'l', -2, None)
+        self.checkequal(2, s, 'find', 'l', None, -2)
+        self.checkequal(0, s, 'find', 'h', None, None)
+
+        self.checkequal(3, s, 'rfind', 'l', None)
+        self.checkequal(3, s, 'rfind', 'l', -2, None)
+        self.checkequal(2, s, 'rfind', 'l', None, -2)
+        self.checkequal(0, s, 'rfind', 'h', None, None)
+
+        self.checkequal(2, s, 'index', 'l', None)
+        self.checkequal(3, s, 'index', 'l', -2, None)
+        self.checkequal(2, s, 'index', 'l', None, -2)
+        self.checkequal(0, s, 'index', 'h', None, None)
+
+        self.checkequal(3, s, 'rindex', 'l', None)
+        self.checkequal(3, s, 'rindex', 'l', -2, None)
+        self.checkequal(2, s, 'rindex', 'l', None, -2)
+        self.checkequal(0, s, 'rindex', 'h', None, None)
+
+        self.checkequal(2, s, 'count', 'l', None)
+        self.checkequal(1, s, 'count', 'l', -2, None)
+        self.checkequal(1, s, 'count', 'l', None, -2)
+        self.checkequal(0, s, 'count', 'x', None, None)
+
+        self.checkequal(True, s, 'endswith', 'o', None)
+        self.checkequal(True, s, 'endswith', 'lo', -2, None)
+        self.checkequal(True, s, 'endswith', 'l', None, -2)
+        self.checkequal(False, s, 'endswith', 'x', None, None)
+
+        self.checkequal(True, s, 'startswith', 'h', None)
+        self.checkequal(True, s, 'startswith', 'l', -2, None)
+        self.checkequal(True, s, 'startswith', 'h', None, -2)
+        self.checkequal(False, s, 'startswith', 'x', None, None)
+
+    def test_find_etc_raise_correct_error_messages(self):
+        # issue 11828
+        s = 'hello'
+        x = 'x'
+        self.assertRaisesRegex(TypeError, r'^find\(', s.find,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^index\(', s.index,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^count\(', s.count,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith,
+                                x, None, None, None)
+        self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith,
+                                x, None, None, None)
+
+        # issue #15534
+        self.checkequal(10, "...\u043c......<", "find", "<")
+
+
+class MixinStrUnicodeTest:
+    # Additional tests that only work with str.
+
+    def test_bug1001011(self):
+        # Make sure join returns a NEW object for single item sequences
+        # involving a subclass.
+        # Make sure that it is of the appropriate type.
+        # Check the optimisation still occurs for standard objects.
+        t = self.type2test
+        class subclass(t):
+            pass
+        s1 = subclass("abcd")
+        s2 = t().join([s1])
+        self.assertIsNot(s1, s2)
+        self.assertIs(type(s2), t)
+
+        s1 = t("abcd")
+        s2 = t().join([s1])
+        self.assertIs(s1, s2)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/time_pxd.pyx cython-0.20.1+1~202203241016-9537/tests/run/time_pxd.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/time_pxd.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/time_pxd.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,64 @@
+# mode: run
+# tag: py3only,pytime
+
+import time
+
+from cpython cimport time as ctime
+
+
+def test_time():
+    """
+    >>> tic1, tic2, tic3 = test_time()
+    >>> tic1 <= tic3  # sanity check
+    True
+    >>> tic1 <= tic2
+    True
+    >>> tic2 <= tic3
+    True
+    """
+    # check that ctime.time() matches time.time() to within call-time tolerance
+    tic1 = time.time()
+    tic2 = ctime.time()
+    tic3 = time.time()
+
+    return tic1, tic2, tic3
+
+
+def test_localtime():
+    """
+    >>> ltp, ltc = test_localtime()
+    >>> ltp.tm_year == ltc['tm_year']  or  (ltp.tm_year, ltc['tm_year'])
+    True
+    >>> ltp.tm_mon == ltc['tm_mon']  or  (ltp.tm_mon, ltc['tm_mon'])
+    True
+    >>> ltp.tm_mday == ltc['tm_mday']  or  (ltp.tm_mday, ltc['tm_mday'])
+    True
+    >>> ltp.tm_hour == ltc['tm_hour']  or  (ltp.tm_hour, ltc['tm_hour'])
+    True
+    >>> ltp.tm_min == ltc['tm_min']  or  (ltp.tm_min, ltc['tm_min'])
+    True
+    >>> ltp.tm_sec == ltc['tm_sec']  or  (ltp.tm_sec, ltc['tm_sec'])
+    True
+    >>> ltp.tm_wday == ltc['tm_wday']  or (ltp.tm_wday, ltc['tm_wday'])
+    True
+    >>> ltp.tm_yday == ltc['tm_yday']  or  (ltp.tm_yday, ltc['tm_yday'])
+    True
+    >>> ltp.tm_isdst == ltc['tm_isdst']  or  (ltp.tm_isdst, ltc['tm_isdst'])
+    True
+    """
+    ltp = time.localtime()
+    ltc = ctime.localtime()
+
+    i = 0
+    while ltp.tm_sec != ltc.tm_sec:
+        # If the time.localtime call is just before the end of a second and the
+        #  ctime.localtime call is just after the beginning of the next second,
+        #  re-call.  This should not occur twice in a row.
+        time.sleep(0.1)
+        ltp = time.localtime()
+        ltc = ctime.localtime()
+        i += 1
+        if i > 10:
+            break
+
+    return ltp, ltc
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tp_new.pyx cython-0.20.1+1~202203241016-9537/tests/run/tp_new.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tp_new.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tp_new.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,6 @@
-# ticket: 808
+# mode: run
+# tag: exttype, tpnew
+# ticket: t808
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tp_new_T454.pyx cython-0.20.1+1~202203241016-9537/tests/run/tp_new_T454.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tp_new_T454.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tp_new_T454.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 454
+# ticket: t454
 
 cimport cython
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tracebacks.pyx cython-0.20.1+1~202203241016-9537/tests/run/tracebacks.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tracebacks.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tracebacks.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,36 @@
+# tag: traceback
+
+def foo1():
+  foo2()
+
+cdef foo2():
+  foo3()
+
+cdef int foo3() except -1:
+  raise RuntimeError('my_message')
+
+def test_traceback(cline_in_traceback=None):
+  """
+  >>> test_traceback()
+  >>> test_traceback(True)
+  >>> test_traceback(False)
+  """
+  if cline_in_traceback is not None:
+    import cython_runtime
+    cython_runtime.cline_in_traceback = cline_in_traceback
+  try:
+    foo1()
+  except:
+    import traceback
+    tb_string = traceback.format_exc()
+    expected = (
+      'tracebacks.pyx',
+      'foo1', 'foo2', 'foo3',
+      'line 4', 'line 7', 'line 10',
+      'my_message')
+    for s in expected:
+      assert s in tb_string, s
+    if cline_in_traceback:
+      assert 'tracebacks.c' in tb_string
+    else:
+      assert 'tracebacks.c' not in tb_string
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/trashcan.pyx cython-0.20.1+1~202203241016-9537/tests/run/trashcan.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/trashcan.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/trashcan.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,148 @@
+# mode: run
+
+cimport cython
+
+
+# Count number of times an object was deallocated twice. This should remain 0.
+cdef int double_deallocations = 0
+def assert_no_double_deallocations():
+    global double_deallocations
+    err = double_deallocations
+    double_deallocations = 0
+    assert not err
+
+
+# Compute x = f(f(f(...(None)...))) nested n times and throw away the result.
+# The real test happens when exiting this function: then a big recursive
+# deallocation of x happens. We are testing two things in the tests below:
+# that Python does not crash and that no double deallocation happens.
+# See also https://github.com/python/cpython/pull/11841
+def recursion_test(f, int n=2**20):
+    x = None
+    cdef int i
+    for i in range(n):
+        x = f(x)
+
+
+@cython.trashcan(True)
+cdef class Recurse:
+    """
+    >>> recursion_test(Recurse)
+    >>> assert_no_double_deallocations()
+    """
+    cdef public attr
+    cdef int deallocated
+
+    def __cinit__(self, x):
+        self.attr = x
+
+    def __dealloc__(self):
+        # Check that we're not being deallocated twice
+        global double_deallocations
+        double_deallocations += self.deallocated
+        self.deallocated = 1
+
+
+cdef class RecurseSub(Recurse):
+    """
+    >>> recursion_test(RecurseSub)
+    >>> assert_no_double_deallocations()
+    """
+    cdef int subdeallocated
+
+    def __dealloc__(self):
+        # Check that we're not being deallocated twice
+        global double_deallocations
+        double_deallocations += self.subdeallocated
+        self.subdeallocated = 1
+
+
+@cython.freelist(4)
+@cython.trashcan(True)
+cdef class RecurseFreelist:
+    """
+    >>> recursion_test(RecurseFreelist)
+    >>> recursion_test(RecurseFreelist, 1000)
+    >>> assert_no_double_deallocations()
+    """
+    cdef public attr
+    cdef int deallocated
+
+    def __cinit__(self, x):
+        self.attr = x
+
+    def __dealloc__(self):
+        # Check that we're not being deallocated twice
+        global double_deallocations
+        double_deallocations += self.deallocated
+        self.deallocated = 1
+
+
+# Subclass of list => uses trashcan by default
+# As long as https://github.com/python/cpython/pull/11841 is not fixed,
+# this does lead to double deallocations, so we skip that check.
+cdef class RecurseList(list):
+    """
+    >>> RecurseList(42)
+    [42]
+    >>> recursion_test(RecurseList)
+    """
+    def __init__(self, x):
+        super().__init__((x,))
+
+
+# Some tests where the trashcan is NOT used. When the trashcan is not used
+# in a big recursive deallocation, the __dealloc__s of the base classes are
+# only run after the __dealloc__s of the subclasses.
+# We use this to detect trashcan usage.
+cdef int base_deallocated = 0
+cdef int trashcan_used = 0
+def assert_no_trashcan_used():
+    global base_deallocated, trashcan_used
+    err = trashcan_used
+    trashcan_used = base_deallocated = 0
+    assert not err
+
+
+cdef class Base:
+    def __dealloc__(self):
+        global base_deallocated
+        base_deallocated = 1
+
+
+# Trashcan disabled by default
+cdef class Sub1(Base):
+    """
+    >>> recursion_test(Sub1, 100)
+    >>> assert_no_trashcan_used()
+    """
+    cdef public attr
+
+    def __cinit__(self, x):
+        self.attr = x
+
+    def __dealloc__(self):
+        global base_deallocated, trashcan_used
+        trashcan_used += base_deallocated
+
+
+@cython.trashcan(True)
+cdef class Middle(Base):
+    cdef public foo
+
+
+# Trashcan disabled explicitly
+@cython.trashcan(False)
+cdef class Sub2(Middle):
+    """
+    >>> recursion_test(Sub2, 1000)
+    >>> assert_no_trashcan_used()
+    """
+    cdef public attr
+
+    def __cinit__(self, x):
+        self.attr = x
+
+    def __dealloc__(self):
+        global base_deallocated, trashcan_used
+        trashcan_used += base_deallocated
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tryexcept.pyx cython-0.20.1+1~202203241016-9537/tests/run/tryexcept.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tryexcept.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tryexcept.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -58,6 +58,36 @@
         i = 3
     return i
 
+
+exceptions = (ValueError, TypeError)
+
+
+def single_except_global_tuple(x):
+    """
+    >>> single_except_global_tuple(None)
+    2
+    >>> single_except_global_tuple(ValueError('test'))
+    3
+    >>> single_except_global_tuple(TypeError('test'))
+    3
+    >>> class TypeErrorSubtype(TypeError): pass
+    >>> single_except_global_tuple(TypeErrorSubtype('test'))
+    3
+    >>> single_except_global_tuple(AttributeError('test'))
+    Traceback (most recent call last):
+    AttributeError: test
+    """
+    cdef int i
+    try:
+        i = 1
+        if x:
+            raise x
+        i = 2
+    except exceptions:
+        i = 3
+    return i
+
+
 def double_except_no_raise(a,b):
     """
     >>> double_except_no_raise(TypeError, ValueError)
@@ -179,6 +209,10 @@
     2
     >>> normal_and_bare_except_raise(ValueError('test'), TypeError)
     3
+    >>> normal_and_bare_except_raise(TypeError('test'), (TypeError, ValueError))
+    2
+    >>> normal_and_bare_except_raise(ValueError('test'), (TypeError, ValueError))
+    2
     >>> normal_and_bare_except_raise(None, TypeError)
     1
     """
@@ -467,3 +501,20 @@
     else:
         i = 5
     return i
+
+
+def try_except_raise_new(initial, catch, throw):
+    """
+    >>> try_except_raise_new(None, TypeError, ValueError)
+    >>> try_except_raise_new(TypeError, IndexError, ValueError)
+    Traceback (most recent call last):
+    TypeError
+    >>> try_except_raise_new(TypeError, TypeError, ValueError)
+    Traceback (most recent call last):
+    ValueError
+    """
+    try:
+        if initial is not None:
+            raise initial
+    except catch:
+        raise throw
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tryfinally.pyx cython-0.20.1+1~202203241016-9537/tests/run/tryfinally.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tryfinally.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tryfinally.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -549,3 +549,21 @@
             del l[0], lobj[0]
             assert all(i == 3 for i in l), l
     return 99
+
+def function_in_finally():
+    """
+    https://github.com/cython/cython/issues/4651 - function definitions in the
+    except copy of the finally clause weren't generated
+
+    >>> function_in_finally()
+    in try
+    in func()
+    finished
+    """
+    try:
+        print('in try')
+    finally:
+        def func():
+            print('in func()')
+        func()
+    print('finished')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tss.pyx cython-0.20.1+1~202203241016-9537/tests/run/tss.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tss.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tss.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,75 @@
+# mode: run
+
+from cpython.pythread cimport *
+
+
+cdef Py_tss_t *pass_py_tss_t_ptr(Py_tss_t *value):
+    return value
+
+
+def tss_create_delete():
+    """
+    >>> tss_create_delete()
+    (True, False)
+    """
+    cdef Py_tss_t tss_key
+    cdef bint after_create, after_delete
+    if PyThread_tss_create(&tss_key) != 0:
+        raise MemoryError()
+    after_create = PyThread_tss_is_created(&tss_key) != 0
+    assert after_create == PyThread_tss_is_created(pass_py_tss_t_ptr(&tss_key))
+    PyThread_tss_delete(&tss_key)
+    after_delete = PyThread_tss_is_created(&tss_key) != 0
+    return (after_create, after_delete)
+
+
+def tss_alloc_free():
+    """
+    >>> tss_alloc_free()
+    False
+    """
+    cdef Py_tss_t *ptr_key
+    cdef bint after_alloc, after_free
+    ptr_key = PyThread_tss_alloc()
+    if ptr_key == NULL:
+        raise MemoryError()
+    after_alloc = PyThread_tss_is_created(ptr_key) != 0
+    PyThread_tss_free(ptr_key)
+    return after_alloc
+
+
+def tss_alloc_create_delete_free():
+    """
+    >>> tss_alloc_create_delete_free()
+    (False, True, False)
+    """
+    cdef Py_tss_t *ptr_key
+    cdef bint after_alloc, after_free
+    ptr_key = PyThread_tss_alloc()
+    if ptr_key == NULL:
+        raise MemoryError()
+    after_alloc = PyThread_tss_is_created(ptr_key) != 0
+    if PyThread_tss_create(ptr_key) != 0:
+        raise MemoryError()
+    after_create = PyThread_tss_is_created(ptr_key) != 0
+    PyThread_tss_delete(ptr_key)
+    after_delete = PyThread_tss_is_created(ptr_key) != 0
+    PyThread_tss_free(ptr_key)
+    return (after_alloc, after_create, after_delete)
+
+
+def tss_set_get():
+    """
+    >>> tss_set_get()
+    1
+    """
+    cdef Py_tss_t tss_key
+    cdef int the_value = 1
+    cdef int ret_value
+    if PyThread_tss_create(&tss_key) != 0:
+        raise MemoryError()
+    if PyThread_tss_get(&tss_key) == NULL:
+        PyThread_tss_set(&tss_key, &the_value)
+    ret_value = (PyThread_tss_get(&tss_key))[0]
+    PyThread_tss_delete(&tss_key)
+    return ret_value
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tuple_constants.pyx cython-0.20.1+1~202203241016-9537/tests/run/tuple_constants.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tuple_constants.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tuple_constants.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -2,6 +2,9 @@
 cimport cython
 
 module_level_tuple = (1,2,3)
+second_module_level_tuple = (1,2,3)  # should be deduplicated to be the same as the first
+string_module_level_tuple = ("1", "2")
+string_module_level_tuple2 = ("1", "2")
 
 def return_module_level_tuple():
     """
@@ -10,6 +13,32 @@
     """
     return module_level_tuple
 
+def test_deduplicated_tuples():
+    """
+    >>> test_deduplicated_tuples()
+    """
+    assert (module_level_tuple is second_module_level_tuple)
+    assert (module_level_tuple is (1,2,3))  # also deduplicated with a function tuple
+    assert (string_module_level_tuple is string_module_level_tuple2)
+    assert (string_module_level_tuple is ("1", "2"))
+
+def func1(arg1, arg2):
+    pass
+
+def func2(arg1, arg2):
+    pass
+
+def test_deduplicated_args():
+    """
+    >>> test_deduplicated_args()
+    """
+    # This is a concern because in large modules *a lot* of similar code objects
+    # are generated often with the same argument names. Therefore it's worth ensuring that
+    # they are correctly deduplicated
+    import sys
+    if not hasattr(sys, "pypy_version_info"):  # test doesn't work on PyPy (which is probably fair enough)
+        assert func1.__code__.co_varnames is func2.__code__.co_varnames
+
 @cython.test_assert_path_exists("//TupleNode",
                                 "//TupleNode[@is_literal = true]")
 @cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
@@ -46,9 +75,9 @@
 def return_nested_tuple():
     """
     >>> return_nested_tuple()
-    (1, (2, 3), (3, (4, 5)))
+    (1, (2, 3), (3, (4, 5), (2, 3, 2, 3)))
     """
-    return (1, (2, 3), (3, (4, 5)))
+    return (1, (2, 3), (3, (4, 5), (2, 3) * 2))
 
 @cython.test_assert_path_exists("//TupleNode",
                                 "//TupleNode[@is_literal = true]")
@@ -71,6 +100,32 @@
     """
     return (1,2)
 
+
+def return_multiplied_constant_tuple(n):
+    """
+    >>> tuples = return_multiplied_constant_tuple(2)
+    >>> type(tuples) is tuple
+    True
+    >>> for t in tuples: print(t)
+    ()
+    (1, 2, 3)
+    (1, 2, 3, 1, 2, 3)
+    (1, 2, 3, 1, 2, 3, 1, 2, 3)
+    (1, 2, 3, 1, 2, 3)
+    (1, 2, 3, 1, 2, 3)
+    ((1, 2, 3, 1, 2, 3), (1, 2, 3), (1, 2, 3, 1, 2, 3))
+    """
+    return (
+        (1, 2, 3) * 0,
+        (1, 2, 3) * 1,
+        (1, 2, 3) * 2,
+        (1, 2, 3) * 3,
+        (1, 2, 3) * 2,
+        (1, 2, 3) * n,
+        ((1, 2, 3) * n, (1, 2, 3), (1, 2, 3) * n),
+    )
+
+
 @cython.test_assert_path_exists("//TupleNode",
                                 "//TupleNode[@is_literal = true]")
 @cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
@@ -106,3 +161,38 @@
     """
     a = eval("1")
     return ('a', a, 'd')
+
+
+def constant_types_comparing_equal():
+    """
+    >>> constant_types_comparing_equal()
+    ((False, False), (0, 0), (0.0, 0.0), (0, False), (False, 0.0), (0, 0.0))
+    """
+    bool_tuple= (False, False)
+    int_tuple = (0, 0)
+    float_tuple = (0.0, 0.0)
+    int_bool = (0, False)
+    bool_float = (False, 0.0)
+    int_float = (0, 0.0)
+
+    assert bool_tuple is (False, False)
+    assert int_tuple is (0, 0)
+    assert bool_tuple == int_tuple
+    assert bool_tuple is not int_tuple
+    assert float_tuple is (0., 0.)
+    assert float_tuple == int_tuple
+    assert float_tuple is not int_tuple
+    assert int_bool is (0, False)
+    assert int_bool == bool_tuple
+    assert int_bool is not bool_tuple
+    assert int_bool is not int_tuple
+    assert bool_float is (False, 0.)
+    assert bool_float == bool_tuple
+    assert bool_float is not bool_tuple
+    assert bool_float is not float_tuple
+    assert int_float is (0, 0.)
+    assert int_float == int_tuple
+    assert int_float is not int_tuple
+    assert int_float is not float_tuple
+
+    return bool_tuple, int_tuple, float_tuple, int_bool, bool_float, int_float
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tuple.pyx cython-0.20.1+1~202203241016-9537/tests/run/tuple.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tuple.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tuple.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -117,9 +117,9 @@
 @cython.test_fail_if_path_exists('//SimpleCallNode//SimpleCallNode')
 def tuple_of_object(ob):
     """
-    >>> tuple(type(1))
+    >>> tuple(type(1))  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: 'type' object is not iterable
+    TypeError: ...type...
     >>> sorted(tuple(set([1, 2, 3])))
     [1, 2, 3]
     """
@@ -139,3 +139,29 @@
     TypeError: ...itera...
     """
     return tuple(tuple(tuple(x)))
+
+
+@cython.test_fail_if_path_exists(
+    "//ExprStatNode//TupleNode",
+    "//ExprStatNode",
+)
+def unused_literals():
+    """
+    >>> unused_literals()
+    """
+    (1, 2, 3)
+    (1, 2, 3 + 4)
+    ("abc", 'def')
+    #(int(), 2, 3)
+
+
+@cython.test_assert_path_exists(
+    "//ExprStatNode",
+    "//ExprStatNode//TupleNode",
+)
+def unused_non_literal():
+    """
+    >>> unused_non_literal()
+    """
+    (set(), None)
+    (range(10), None)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tupleunpack_T298.pyx cython-0.20.1+1~202203241016-9537/tests/run/tupleunpack_T298.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tupleunpack_T298.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tupleunpack_T298.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 298
+# ticket: t298
 
 """
 >>> func()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/tupleunpack_T712.pyx cython-0.20.1+1~202203241016-9537/tests/run/tupleunpack_T712.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/tupleunpack_T712.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/tupleunpack_T712.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 712
+# ticket: t712
 
 def single_from_string():
     """
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typeddefaultargT373.pyx cython-0.20.1+1~202203241016-9537/tests/run/typeddefaultargT373.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/typeddefaultargT373.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typeddefaultargT373.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 373
+# ticket: t373
 
 import math
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typedfieldbug_T303.pyx cython-0.20.1+1~202203241016-9537/tests/run/typedfieldbug_T303.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/typedfieldbug_T303.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typedfieldbug_T303.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 303
+# ticket: t303
 
 __doc__ = """
 >>> try: readonly()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typed_slice.pyx cython-0.20.1+1~202203241016-9537/tests/run/typed_slice.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/typed_slice.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typed_slice.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,10 @@
 # mode: run
-# tag: list, tuple, slice
+# tag: list, tuple, slice, slicing
 
+cimport cython
+
+
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list(list l, int start, int stop):
     """
     >>> slice_list([1,2,3,4], 1, 3)
@@ -22,6 +26,7 @@
     """
     return l[start:stop]
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_start(list l, int start):
     """
     >>> slice_list_start([1,2,3,4], 1)
@@ -44,6 +49,7 @@
     return l[start:]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_stop(list l, int stop):
     """
     >>> slice_list_stop([1,2,3,4], 3)
@@ -68,6 +74,7 @@
     return l[:stop]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_copy(list l):
     """
     >>> slice_list_copy([])
@@ -78,6 +85,7 @@
     return l[:]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_tuple_copy(tuple l):
     """
     >>> slice_tuple_copy(())
@@ -88,6 +96,7 @@
     return l[:]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_tuple(tuple t, int start, int stop):
     """
     >>> slice_tuple((1,2,3,4), 1, 3)
@@ -110,6 +119,7 @@
     return t[start:stop]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_tuple_start(tuple t, int start):
     """
     >>> slice_tuple_start((1,2,3,4), 1)
@@ -129,6 +139,8 @@
     """
     return t[start:]
 
+
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_tuple_stop(tuple t, int stop):
     """
     >>> slice_tuple_stop((1,2,3,4), 3)
@@ -147,6 +159,7 @@
     return t[:stop]
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_assign_list(list l):
     """
     >>> l = [1,2,3,4]
@@ -158,6 +171,7 @@
     return l
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_assign_tuple(list l):
     """
     >>> l = [1,2,3,4]
@@ -169,6 +183,7 @@
     return l
 
 
+@cython.test_fail_if_path_exists("//CondExprNode")
 def slice_list_assign(list l, value):
     """
     >>> l = [1,2,3,4]
@@ -203,3 +218,392 @@
     cdef bytes slice_val = s[1:6]
     s = slice_val
     return s[1:3].decode(u'ASCII')
+
+
+# Readers will find the common boilerplate in the tests below:
+#     >>> l = [1,2,3,4,5]
+#     >>> t = tuple(l)
+#     >>> b = ''.join(map(str, l)).encode('ASCII')
+#     >>> u = b.decode('ASCII')
+#     >>> o = (l, t, b, u)
+#     >>> n = ('list', 'tuple', 'bytes', 'unicode')
+#     >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+#     >>> r = lambda i, *a: '%s[%s] -> %s' % (n[i], ':'.join(map(repr, a)), FUNCTION_NAME(o[i], *a))
+# Originally, this was planned to be a basic iteration over
+#   the various object types contained within the slicable fused
+#   type, but Python2 -> Python3 semantics changed the class names
+#   and string representations used for raw bytes and unicode.
+# As a result, we dynamically adjust the printed string output
+#   for each test in order to ensure consistent results when running
+#   both Python2 and Python3.
+
+ctypedef fused slicable:
+    list
+    tuple
+    bytes
+    unicode
+
+
+@cython.test_assert_path_exists("//SliceIndexNode//CondExprNode")
+def slice_fused_type_start(slicable seq, start):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[%r:] -> %s' % (n[i], s, p(slice_fused_type_start(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (0, len(l) - 1, len(l), -1, -len(l), None):
+    ...         print(r(i, s))
+    ... 
+    list[0:] -> [1, 2, 3, 4, 5]
+    list[4:] -> [5]
+    list[5:] -> []
+    list[-1:] -> [5]
+    list[-5:] -> [1, 2, 3, 4, 5]
+    list[None:] -> [1, 2, 3, 4, 5]
+    tuple[0:] -> (1, 2, 3, 4, 5)
+    tuple[4:] -> (5,)
+    tuple[5:] -> ()
+    tuple[-1:] -> (5,)
+    tuple[-5:] -> (1, 2, 3, 4, 5)
+    tuple[None:] -> (1, 2, 3, 4, 5)
+    bytes[0:] -> 12345
+    bytes[4:] -> 5
+    bytes[5:] -> 
+    bytes[-1:] -> 5
+    bytes[-5:] -> 12345
+    bytes[None:] -> 12345
+    unicode[0:] -> 12345
+    unicode[4:] -> 5
+    unicode[5:] -> 
+    unicode[-1:] -> 5
+    unicode[-5:] -> 12345
+    unicode[None:] -> 12345
+    """
+    obj = seq[start:]
+    return obj
+
+
+@cython.test_assert_path_exists("//SliceIndexNode//CondExprNode")
+def slice_fused_type_stop(slicable seq, stop):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[:%r] -> %s' % (n[i], s, p(slice_fused_type_stop(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (0, len(l) - 1, len(l), -1, -len(l), None):
+    ...         print(r(i, s))
+    ... 
+    list[:0] -> []
+    list[:4] -> [1, 2, 3, 4]
+    list[:5] -> [1, 2, 3, 4, 5]
+    list[:-1] -> [1, 2, 3, 4]
+    list[:-5] -> []
+    list[:None] -> [1, 2, 3, 4, 5]
+    tuple[:0] -> ()
+    tuple[:4] -> (1, 2, 3, 4)
+    tuple[:5] -> (1, 2, 3, 4, 5)
+    tuple[:-1] -> (1, 2, 3, 4)
+    tuple[:-5] -> ()
+    tuple[:None] -> (1, 2, 3, 4, 5)
+    bytes[:0] -> 
+    bytes[:4] -> 1234
+    bytes[:5] -> 12345
+    bytes[:-1] -> 1234
+    bytes[:-5] -> 
+    bytes[:None] -> 12345
+    unicode[:0] -> 
+    unicode[:4] -> 1234
+    unicode[:5] -> 12345
+    unicode[:-1] -> 1234
+    unicode[:-5] -> 
+    unicode[:None] -> 12345
+    """
+    obj = seq[:stop]
+    return obj
+
+
+@cython.test_assert_path_exists("//SliceIndexNode//CondExprNode")
+def slice_fused_type_start_and_stop(slicable seq, start, stop):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, t, s: '%s[%r:%r] -> %s' % (n[i], t, s, p(slice_fused_type_start_and_stop(o[i], t, s)))
+    >>> for i in range(len(o)): 
+    ...     for start, stop in ((0, len(l)), (0, None), (None, len(l)),
+    ...                         (-len(l), 0), (1, 0), (0, 1)):
+    ...         print(r(i, start, stop))
+    ... 
+    list[0:5] -> [1, 2, 3, 4, 5]
+    list[0:None] -> [1, 2, 3, 4, 5]
+    list[None:5] -> [1, 2, 3, 4, 5]
+    list[-5:0] -> []
+    list[1:0] -> []
+    list[0:1] -> [1]
+    tuple[0:5] -> (1, 2, 3, 4, 5)
+    tuple[0:None] -> (1, 2, 3, 4, 5)
+    tuple[None:5] -> (1, 2, 3, 4, 5)
+    tuple[-5:0] -> ()
+    tuple[1:0] -> ()
+    tuple[0:1] -> (1,)
+    bytes[0:5] -> 12345
+    bytes[0:None] -> 12345
+    bytes[None:5] -> 12345
+    bytes[-5:0] -> 
+    bytes[1:0] -> 
+    bytes[0:1] -> 1
+    unicode[0:5] -> 12345
+    unicode[0:None] -> 12345
+    unicode[None:5] -> 12345
+    unicode[-5:0] -> 
+    unicode[1:0] -> 
+    unicode[0:1] -> 1
+    """
+    obj = seq[start:stop]
+    return obj
+
+
+def slice_fused_type_step(slicable seq, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s: '%s[::%r] -> %s' % (n[i], s, p(slice_fused_type_step(o[i], s)))
+    >>> for i in range(len(o)):
+    ...     for s in (1, -1, 2, -3, 5, -5, None):
+    ...         print(r(i, s))
+    ... 
+    list[::1] -> [1, 2, 3, 4, 5]
+    list[::-1] -> [5, 4, 3, 2, 1]
+    list[::2] -> [1, 3, 5]
+    list[::-3] -> [5, 2]
+    list[::5] -> [1]
+    list[::-5] -> [5]
+    list[::None] -> [1, 2, 3, 4, 5]
+    tuple[::1] -> (1, 2, 3, 4, 5)
+    tuple[::-1] -> (5, 4, 3, 2, 1)
+    tuple[::2] -> (1, 3, 5)
+    tuple[::-3] -> (5, 2)
+    tuple[::5] -> (1,)
+    tuple[::-5] -> (5,)
+    tuple[::None] -> (1, 2, 3, 4, 5)
+    bytes[::1] -> 12345
+    bytes[::-1] -> 54321
+    bytes[::2] -> 135
+    bytes[::-3] -> 52
+    bytes[::5] -> 1
+    bytes[::-5] -> 5
+    bytes[::None] -> 12345
+    unicode[::1] -> 12345
+    unicode[::-1] -> 54321
+    unicode[::2] -> 135
+    unicode[::-3] -> 52
+    unicode[::5] -> 1
+    unicode[::-5] -> 5
+    unicode[::None] -> 12345
+    >>> for v in o:
+    ...     try: slice_fused_type_step(v, 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_step(v, v)
+    ...     except TypeError: pass
+    """
+    obj = seq[::step]
+    return obj
+
+
+def slice_fused_type_start_and_step(slicable seq, start, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t: '%s[%r::%r] -> %s' % (n[i], s, t, p(slice_fused_type_start_and_step(o[i], s, t)))
+    >>> for i in range(len(o)):
+    ...     for start, step in ((0, 1), (0, -1), (1, 1), (1, -1),
+    ...                         (None, 1), (None, -1), (None, None),
+    ...                         (1, 2), (len(l), -2), (len(l), len(l))):
+    ...         print(r(i, start, step))
+    ... 
+    list[0::1] -> [1, 2, 3, 4, 5]
+    list[0::-1] -> [1]
+    list[1::1] -> [2, 3, 4, 5]
+    list[1::-1] -> [2, 1]
+    list[None::1] -> [1, 2, 3, 4, 5]
+    list[None::-1] -> [5, 4, 3, 2, 1]
+    list[None::None] -> [1, 2, 3, 4, 5]
+    list[1::2] -> [2, 4]
+    list[5::-2] -> [5, 3, 1]
+    list[5::5] -> []
+    tuple[0::1] -> (1, 2, 3, 4, 5)
+    tuple[0::-1] -> (1,)
+    tuple[1::1] -> (2, 3, 4, 5)
+    tuple[1::-1] -> (2, 1)
+    tuple[None::1] -> (1, 2, 3, 4, 5)
+    tuple[None::-1] -> (5, 4, 3, 2, 1)
+    tuple[None::None] -> (1, 2, 3, 4, 5)
+    tuple[1::2] -> (2, 4)
+    tuple[5::-2] -> (5, 3, 1)
+    tuple[5::5] -> ()
+    bytes[0::1] -> 12345
+    bytes[0::-1] -> 1
+    bytes[1::1] -> 2345
+    bytes[1::-1] -> 21
+    bytes[None::1] -> 12345
+    bytes[None::-1] -> 54321
+    bytes[None::None] -> 12345
+    bytes[1::2] -> 24
+    bytes[5::-2] -> 531
+    bytes[5::5] -> 
+    unicode[0::1] -> 12345
+    unicode[0::-1] -> 1
+    unicode[1::1] -> 2345
+    unicode[1::-1] -> 21
+    unicode[None::1] -> 12345
+    unicode[None::-1] -> 54321
+    unicode[None::None] -> 12345
+    unicode[1::2] -> 24
+    unicode[5::-2] -> 531
+    unicode[5::5] -> 
+    >>> for o in (l, t, b):
+    ...     try: slice_fused_type_start_and_step(o, 0, 0)
+    ...     except ValueError: pass
+    """
+    obj = seq[start::step]
+    return obj
+
+
+def slice_fused_type_stop_and_step(slicable seq, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t: '%s[:%r:%r] -> %s' % (n[i], s, t, p(slice_fused_type_stop_and_step(o[i], s, t)))
+    >>> for i in range(len(o)):
+    ...     for stop, step in ((len(l), 1), (len(l), None), (None, 1),
+    ...                        (len(l), -1), (len(l) - 1, 2), (len(l), -2),
+    ...                        (len(l), len(l))):
+    ...         print(r(i, stop, step))
+    ... 
+    list[:5:1] -> [1, 2, 3, 4, 5]
+    list[:5:None] -> [1, 2, 3, 4, 5]
+    list[:None:1] -> [1, 2, 3, 4, 5]
+    list[:5:-1] -> []
+    list[:4:2] -> [1, 3]
+    list[:5:-2] -> []
+    list[:5:5] -> [1]
+    tuple[:5:1] -> (1, 2, 3, 4, 5)
+    tuple[:5:None] -> (1, 2, 3, 4, 5)
+    tuple[:None:1] -> (1, 2, 3, 4, 5)
+    tuple[:5:-1] -> ()
+    tuple[:4:2] -> (1, 3)
+    tuple[:5:-2] -> ()
+    tuple[:5:5] -> (1,)
+    bytes[:5:1] -> 12345
+    bytes[:5:None] -> 12345
+    bytes[:None:1] -> 12345
+    bytes[:5:-1] -> 
+    bytes[:4:2] -> 13
+    bytes[:5:-2] -> 
+    bytes[:5:5] -> 1
+    unicode[:5:1] -> 12345
+    unicode[:5:None] -> 12345
+    unicode[:None:1] -> 12345
+    unicode[:5:-1] -> 
+    unicode[:4:2] -> 13
+    unicode[:5:-2] -> 
+    unicode[:5:5] -> 1
+    >>> for v in o:
+    ...     try: slice_fused_type_stop_and_step(v, len(l), 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_stop_and_step(v, len(l), v)
+    ...     except TypeError: pass
+    """
+    obj = seq[:stop:step]
+    return obj
+
+
+def slice_fused_type_all(slicable seq, start, stop, step):
+    """
+    >>> l = [1,2,3,4,5]
+    >>> t = tuple(l)
+    >>> b = ''.join(map(str, l)).encode('ASCII')
+    >>> u = b.decode('ASCII')
+    >>> o = (l, t, b, u)
+    >>> n = ('list', 'tuple', 'bytes', 'unicode')
+    >>> p = lambda o: o.decode() if isinstance(o, type(b)) else str(o)
+    >>> r = lambda i, s, t, e: '%s[%r:%r:%r] -> %s' % (n[i], s, t, e, p(slice_fused_type_all(o[i], s, t, e)))
+    >>> for i in range(len(o)):
+    ...     for args in ((0, len(l), 1), (len(l), 0, -1), (None, len(l), 1),
+    ...                  (len(l), None, -1), (-len(l), len(l), None), (None, None, None),
+    ...                  (1, 3, 2), (len(l), 1, -3), (len(l), 0, 1)):
+    ...         print(r(i, *args))
+    ... 
+    list[0:5:1] -> [1, 2, 3, 4, 5]
+    list[5:0:-1] -> [5, 4, 3, 2]
+    list[None:5:1] -> [1, 2, 3, 4, 5]
+    list[5:None:-1] -> [5, 4, 3, 2, 1]
+    list[-5:5:None] -> [1, 2, 3, 4, 5]
+    list[None:None:None] -> [1, 2, 3, 4, 5]
+    list[1:3:2] -> [2]
+    list[5:1:-3] -> [5]
+    list[5:0:1] -> []
+    tuple[0:5:1] -> (1, 2, 3, 4, 5)
+    tuple[5:0:-1] -> (5, 4, 3, 2)
+    tuple[None:5:1] -> (1, 2, 3, 4, 5)
+    tuple[5:None:-1] -> (5, 4, 3, 2, 1)
+    tuple[-5:5:None] -> (1, 2, 3, 4, 5)
+    tuple[None:None:None] -> (1, 2, 3, 4, 5)
+    tuple[1:3:2] -> (2,)
+    tuple[5:1:-3] -> (5,)
+    tuple[5:0:1] -> ()
+    bytes[0:5:1] -> 12345
+    bytes[5:0:-1] -> 5432
+    bytes[None:5:1] -> 12345
+    bytes[5:None:-1] -> 54321
+    bytes[-5:5:None] -> 12345
+    bytes[None:None:None] -> 12345
+    bytes[1:3:2] -> 2
+    bytes[5:1:-3] -> 5
+    bytes[5:0:1] -> 
+    unicode[0:5:1] -> 12345
+    unicode[5:0:-1] -> 5432
+    unicode[None:5:1] -> 12345
+    unicode[5:None:-1] -> 54321
+    unicode[-5:5:None] -> 12345
+    unicode[None:None:None] -> 12345
+    unicode[1:3:2] -> 2
+    unicode[5:1:-3] -> 5
+    unicode[5:0:1] -> 
+    >>> for v in o:
+    ...     try: slice_fused_type_stop_and_step(v, len(l), 0)
+    ...     except ValueError: pass
+    ...     try: slice_fused_type_stop_and_step(v, len(l), v)
+    ...     except TypeError: pass
+    """
+    obj = seq[start:stop:step]
+    return obj
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/type_inference.pyx cython-0.20.1+1~202203241016-9537/tests/run/type_inference.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/type_inference.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/type_inference.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -137,15 +137,21 @@
     a = 3
     a = 4
     a = 5
-    assert typeof(a) == "long"
+    assert typeof(a) == "long", typeof(a)
     b = a
     b = 3.1
     b = 3.14159
-    assert typeof(b) == "double"
+    assert typeof(b) == "double", typeof(b)
     c = a
     c = b
     c = [1,2,3]
-    assert typeof(c) == "Python object"
+    assert typeof(c) == "Python object", typeof(c)
+    d = b'abc'
+    d = bytes()
+    d = bytes(b'xyz')
+    d = None
+    assert typeof(d) == "bytes object", typeof(d)
+
 
 def arithmetic():
     """
@@ -270,6 +276,9 @@
     assert typeof(e) == "double"
 
 def cascaded_assignment():
+    """
+    >>> cascaded_assignment()
+    """
     a = b = c = d = 1.0
     assert typeof(a) == "double"
     assert typeof(b) == "double"
@@ -278,6 +287,36 @@
     e = a + b + c + d
     assert typeof(e) == "double"
 
+
+def unpacking(x):
+    """
+    >>> unpacking(0)
+    """
+    a, b, c, (d, e) = x, 1, 2.0, [3, [5, 6]]
+    assert typeof(a) == "Python object", typeof(a)
+    assert typeof(b) == "long", typeof(b)
+    assert typeof(c) == "double", typeof(c)
+    assert typeof(d) == "long", typeof(d)
+    assert typeof(e) == "list object", typeof(e)
+
+
+def star_unpacking(*x):
+    """
+    >>> star_unpacking(1, 2)
+    """
+    a, b = x
+    c, *d = x
+    *e, f = x
+    *g, g = x  # re-assignment
+    assert typeof(a) == "Python object", typeof(a)
+    assert typeof(b) == "Python object", typeof(b)
+    assert typeof(c) == "Python object", typeof(c)
+    assert typeof(d) == "list object", typeof(d)
+    assert typeof(e) == "list object", typeof(e)
+    assert typeof(f) == "Python object", typeof(f)
+    assert typeof(g) == "Python object", typeof(f)
+
+
 def increment():
     """
     >>> increment()
@@ -486,6 +525,11 @@
     for j in range(10):
         res = -j
     assert typeof(j) == "Python object", typeof(j)
+    h = 1
+    res = abs(h)
+    assert typeof(h) == "Python object", typeof(h)
+    cdef int c_int = 1
+    assert typeof(abs(c_int)) == "int", typeof(abs(c_int))
 
 @infer_types(None)
 def safe_c_functions():
@@ -722,3 +766,39 @@
             d = enum_y
             c = d
             return typeof(a), typeof(b), typeof(c), typeof(d)
+
+cdef class WithMethods:
+    cdef int offset
+    def __init__(self, offset):
+        self.offset = offset
+    cpdef int one_arg(self, int x):
+        return x + self.offset
+    cpdef int default_arg(self, int x, int y=0):
+        return x + y + self.offset
+
+def test_bound_methods():
+  """
+  >>> test_bound_methods()
+  """
+  o = WithMethods(10)
+  assert typeof(o) == 'WithMethods', typeof(o)
+
+  one_arg = o.one_arg
+  assert one_arg(2) == 12, one_arg(2)
+
+  default_arg = o.default_arg
+  assert default_arg(2) == 12, default_arg(2)
+  assert default_arg(2, 3) == 15, default_arg(2, 2)
+
+def test_builtin_max():
+    """
+    # builtin max is slightly complicated because it gets transformed to EvalWithTempExprNode
+    # See https://github.com/cython/cython/issues/4155
+    >>> test_builtin_max()
+    """
+    class C:
+        a = 2
+        def get_max(self):
+            a = max(self.a, self.a)
+            assert typeof(a) == "Python object", typeof(a)
+    C().get_max()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/type_inference_T768_cpp.pyx cython-0.20.1+1~202203241016-9537/tests/run/type_inference_T768_cpp.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/type_inference_T768_cpp.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/type_inference_T768_cpp.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,6 @@
 # mode: run
 # tag: cpp
-# ticket: 768
+# ticket: t768
 from cython cimport typeof
 
 cdef extern from "shapes.h" namespace "shapes":
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/type_inference_T768.pyx cython-0.20.1+1~202203241016-9537/tests/run/type_inference_T768.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/type_inference_T768.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/type_inference_T768.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,5 @@
 # mode: run
-# ticket: 768
+# ticket: t768
 from cython cimport typeof
 
 def type_inference_del_int():
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/type_slots_int_long_T287.pyx cython-0.20.1+1~202203241016-9537/tests/run/type_slots_int_long_T287.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/type_slots_int_long_T287.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/type_slots_int_long_T287.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 287
+# ticket: t287
 
 __doc__ = u"""
 >>> print( "%d" % Int() )
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typetest_T417.pyx cython-0.20.1+1~202203241016-9537/tests/run/typetest_T417.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/typetest_T417.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typetest_T417.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 417
+# ticket: t417
 #cython: autotestdict=True
 
 cdef class Foo:
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typing_module_cy.pyx cython-0.20.1+1~202203241016-9537/tests/run/typing_module_cy.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/typing_module_cy.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typing_module_cy.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,38 @@
+# mode: run
+
+import cython
+
+try:
+    import typing
+    from typing import List
+    from typing import Set as _SET_
+except ImportError:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    cdef typing.Dict[int, float] a = {}
+    cdef List[int] b = []
+    cdef _SET_[object] c = set()
+
+    print(cython.typeof(a))
+    print(cython.typeof(b))
+    print(cython.typeof(c))
+
+cdef class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    cdef int regular
+    cdef typing.ClassVar[int] cls
+    cls = 5
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/typing_module.py cython-0.20.1+1~202203241016-9537/tests/run/typing_module.py
--- cython-0.20.1+1~201611251650-6686/tests/run/typing_module.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/typing_module.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,42 @@
+# mode: run
+# tag: pure3.6
+
+from __future__ import print_function
+
+import cython
+
+try:
+    import typing
+    from typing import List
+    from typing import Set as _SET_
+except ImportError:
+    pass  # this should allow Cython to interpret the directives even when the module doesn't exist
+
+
+def test_subscripted_types():
+    """
+    >>> test_subscripted_types()
+    dict object
+    list object
+    set object
+    """
+    a: typing.Dict[int, float] = {}
+    b: List[int] = []
+    c: _SET_[object] = set()
+
+    print(cython.typeof(a) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(b) + (" object" if not cython.compiled else ""))
+    print(cython.typeof(c) + (" object" if not cython.compiled else ""))
+
+@cython.cclass
+class TestClassVar:
+    """
+    >>> TestClassVar.cls
+    5
+    >>> TestClassVar.regular  # doctest: +IGNORE_EXCEPTION_DETAIL
+    Traceback (most recent call last):
+        ...
+    AttributeError:
+    """
+    regular: int
+    cls: typing.ClassVar[int] = 5  # this is a little redundant really because the assignment ensures it
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unbound_special_methods.pyx cython-0.20.1+1~202203241016-9537/tests/run/unbound_special_methods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unbound_special_methods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unbound_special_methods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -14,11 +14,11 @@
     "//AttributeNode[@entry.cname = 'PyUnicode_Contains']")
 def unicode_contains(unicode s, substring):
     """
-    >>> unicode_contains(text, 'fl')
+    >>> unicode_contains(text, u'fl')
     True
-    >>> unicode_contains(text, 'XYZ')
+    >>> unicode_contains(text, u'XYZ')
     False
-    >>> unicode_contains(None, 'XYZ')
+    >>> unicode_contains(None, u'XYZ')
     Traceback (most recent call last):
     AttributeError: 'NoneType' object has no attribute '__contains__'
     """
@@ -32,11 +32,11 @@
     "//NameNode[@entry.cname = 'PyUnicode_Contains']")
 def unicode_contains_unbound(unicode s, substring):
     """
-    >>> unicode_contains_unbound(text, 'fl')
+    >>> unicode_contains_unbound(text, u'fl')
     True
-    >>> unicode_contains_unbound(text, 'XYZ')
+    >>> unicode_contains_unbound(text, u'XYZ')
     False
-    >>> unicode_contains_unbound(None, 'XYZ')   # doctest: +ELLIPSIS
+    >>> unicode_contains_unbound(None, u'XYZ')   # doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: descriptor '__contains__' requires a '...' object but received a 'NoneType'
     """
@@ -46,17 +46,17 @@
 cdef class UnicodeSubclass(unicode):
     """
     >>> u = UnicodeSubclass(text)
-    >>> 'fl' in u
+    >>> u'fl' in u
     False
-    >>> 'XYZ' in u
+    >>> u'XYZ' in u
     True
-    >>> u.method('fl')
+    >>> u.method(u'fl')
     False
-    >>> u.method('XYZ')
+    >>> u.method(u'XYZ')
     True
-    >>> u.operator('fl')
+    >>> u.operator(u'fl')
     False
-    >>> u.operator('XYZ')
+    >>> u.operator(u'XYZ')
     True
     """
     def __contains__(self, substring):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_formatting.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicode_formatting.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_formatting.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_formatting.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,60 @@
+# mode: run
+# tag: stringformat
+
+from __future__ import unicode_literals
+
+
+def ascii_format(a, int b, list c):
+    """
+    >>> print(ascii_format('x', 2, [1]))
+    -'x'-2-[1]-
+    """
+    return '-%a-%a-%a-' % (a, b, c)
+
+
+def repr_format(a, int b, list c):
+    """
+    >>> print(repr_format('x', 2, [1]))
+    -'x'-2-[1]-
+    """
+    return '-%r-%r-%r-' % (a, b, c)
+
+
+def str_format(a, int b, list c):
+    """
+    >>> print(str_format('x', 2, [1]))
+    -x-2-[1]-
+    """
+    return '-%s-%s-%s-' % (a, b, c)
+
+
+def mix_format(a, int b, list c):
+    """
+    >>> print(mix_format('x', 2, [1]))
+    -x-2-[1]-
+    """
+    return '-%s-%r-%a-' % (a, b, c)
+
+
+class PySubtype(unicode):
+    def __rmod__(self, other):
+        return f'PyRMOD({self}, {other})'
+
+
+cdef class ExtSubtype(unicode):
+    def __rmod__(self, other):
+        return f'ExtRMOD({self}, {other})'
+
+
+def subtypes():
+    """
+    >>> py, ext = subtypes()
+    >>> print(py)
+    PyRMOD(PySub, -%s-)
+    >>> print(ext)
+    ExtRMOD(ExtSub, -%s-)
+    """
+    return [
+        '-%s-' % PySubtype("PySub"),
+        '-%s-' % ExtSubtype("ExtSub"),
+    ]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicodefunction.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicodefunction.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicodefunction.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicodefunction.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,6 +1,11 @@
+# mode: run
+# tag: unicode
+
 __doc__ = u"""
    >>> u('test')
    u'test'
+   >>> e
+   u''
    >>> z
    u'test'
    >>> c('testing')
@@ -16,25 +21,63 @@
 #   u'testing a C subtype'
 """
 
+
+cimport cython
+
 import sys
 if sys.version_info[0] >= 3:
     __doc__ = __doc__.replace(u" u'", u" '")
 
 u = unicode
+e = unicode()
 z = unicode(u'test')
 
+
 def c(string):
     return unicode(string)
 
+
 class subu(unicode):
     pass
 
+
 def sub(string):
     return subu(string)
 
+
 #cdef class csubu(unicode):
 #    pass
 
+
 #def csub(string):
 #    return csubu(string)
 
+
+@cython.test_fail_if_path_exists("//SimpleCallNode")
+@cython.test_assert_path_exists("//PythonCapiCallNode")
+def typed(unicode s):
+    """
+    >>> print(typed(None))
+    None
+    >>> type(typed(None)) is u or type(typed(None))
+    True
+    >>> print(typed(u'abc'))
+    abc
+    >>> type(typed(u'abc')) is u or type(typed(u'abc'))
+    True
+    """
+    return unicode(s)
+
+
+@cython.test_fail_if_path_exists(
+    "//SimpleCallNode",
+    "//PythonCapiCallNode",
+)
+def typed_not_none(unicode s not None):
+    """
+    >>> print(typed(u'abc'))
+    abc
+    >>> type(typed(u'abc')) is u or type(typed(u'abc'))
+    True
+    """
+    return unicode(s)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers_import.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers_import.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers_import.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers_import.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# cython: language_level = 3
+# mode: compile
+# tag: pep3131
+
+# compile only test since there's no way to get
+# it to import another test module at runtime
+
+# this test looks at [c]importing unicode stuff
+from unicode_identifiers cimport Fα1, Γναμε2
+cimport unicode_identifiers
+from unicode_identifiers cimport Γναμε2 as Γναμε3
+
+from unicode_identifiers import NormalClassΓΓ
+from unicode_identifiers import NormalClassΓΓ as NörmalCläss
+
+
+cdef class C(unicode_identifiers.Γναμε2):
+    pass
+
+cdef class D(Γναμε2):
+    pass
+
+cdef class E(Γναμε3):
+    pass
+
+def f():
+    Fα1()
+    unicode_identifiers.Fα1()
+
+
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers_normalization.srctree cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers_normalization.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers_normalization.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers_normalization.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+# mode: run
+# tag: pure3.0, pep3131
+
+PYTHON build_tests.py
+# show behaviour in Python mode
+PYTHON -m doctest test0.py
+PYTHON -m doctest test1.py
+PYTHON -m doctest test2.py
+
+PYTHON setup.py build_ext --inplace
+# test in Cython mode
+PYTHON -c "import doctest; import test0 as m; exit(doctest.testmod(m)[0])"
+PYTHON -c "import doctest; import test1 as m; exit(doctest.testmod(m)[0])"
+PYTHON -c "import doctest; import test2 as m; exit(doctest.testmod(m)[0])"
+
+########## setup.py #########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize("test*.py"),
+)
+
+######### build_tests.py ########
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import sys
+import unicodedata
+
+# a few pairs of unicode strings that should be equivalent after normalization
+string_pairs = [("fi", "fi"), # ligature and two letters
+                ("a\u0301", '\u00e1'), # a with acute accent with combining character or as 1 character
+                ("α\u0334\u0362", "α\u0362\u0334") # alpha with a pair of combining characters
+                    # in a different order. No single character to normalize to
+                ]
+
+# Show that the pairs genuinely aren't equal before normalization
+for sp in string_pairs:
+    assert sp[0] != sp[1]
+    assert unicodedata.normalize('NFKC', sp[0]) == unicodedata.normalize('NFKC', sp[1])
+    
+# some code that accesses the identifiers through the two different names
+#  contains doctests
+example_code = [
+"""
+class C:
+    '''
+    >>> C().get()
+    True
+    '''
+    def __init__(self):
+        self.{0} = True
+    def get(self):
+        return self.{1}
+""", """
+def pass_through({0}):
+    '''
+    >>> pass_through(True)
+    True
+    '''
+    return {1}
+""", """
+import cython
+{0} = True
+def test():
+    '''
+    >>> test()
+    True
+    '''
+    return {1}
+"""]
+
+from io import open
+
+for idx, (code, strings) in enumerate(zip(example_code, string_pairs)):
+    with open("test{0}.py".format(idx), "w", encoding="utf8") as f:
+        code = code.format(*strings)
+        f.write("# -*- coding: utf-8 -*-\n")
+        # The code isn't Py2 compatible. Only write actual code in Py3+.
+        if sys.version_info[0] > 2:
+            f.write(code)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers.pxd cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers.pxd
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers.pxd	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers.pxd	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# cython: language_level=3
+
+cdef Fα1()
+cdef class Γναμε2:
+    cdef public int α
+    cdef boring_cdef(self)
+    cdef εxciting_cdef(self)
+    cpdef boring_cpdef(self)
+    cpdef εxciting_cpdef(self)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_identifiers.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_identifiers.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+# mode: run
+# tag: pep3131, traceback
+
+# cython: language_level=3
+
+# Code with unicode identifiers can be compiled with Cython running either Python 2 or 3.
+# However Python access to unicode identifiers is only possible in Python 3. In Python 2
+# it's only really safe to use the unicode identifiers for purely Cython interfaces
+# (although this isn't enforced...). Therefore the majority of the doctests are
+# Python3 only and only a limited set are run in Python2.
+# This is controlled by putting the Python3 only tests in the module __doc__ attribute
+# Most of the individual function and class docstrings are only present as a compile test
+
+cimport cython
+
+import sys
+
+
+if sys.version_info[0] > 2:
+    __doc__ = u"""
+    >>> f()()
+    2
+    >>> f().__name__
+    'nεsted'
+
+    The test is mainly to see if the traceback is generated correctly
+    >>> print_traceback_name()
+    unicode_identifiers.Fα1
+
+    Just check that a cpdef function is callable
+    >>> Fα3()
+    1
+
+    >>> Γναμε2.ναμε3
+    1
+    >>> x = Γναμε2()
+    >>> print(x.α)
+    100
+    >>> x.α = 200
+    >>> print(x.α)
+    200
+
+    >>> B().Ƒ()
+    >>> C().Ƒ()
+
+    Test generation of locals()
+    >>> sorted(Γναμε2().boring_function(1,2).keys())
+    ['self', 'somevalue', 'x', 'ναμε5', 'ναμε6']
+
+    >>> Γναμε2().boring_cpdef() - Γναμε2().εxciting_cpdef()
+    0
+    >>> function_taking_fancy_argument(Γναμε2()).ναμε3
+    1
+    >>> metho_function_taking_fancy_argument(Γναμε2()).ναμε3
+    1
+    >>> NormalClassΓΓ().ναμε
+    10
+    >>> NormalClassΓΓ().εxciting_function(None).__qualname__
+    'NormalClassΓΓ.εxciting_function..nestεd'
+
+    Do kwargs work?
+    >>> unicode_kwarg(αrγ=5)
+    5
+    >>> unicode_kwarg_from_cy()
+    1
+
+    Normalization of attributes
+    (The cdef class version is testable in Python 2 too)
+    >>> NormalizeAttrPy().get()
+    5
+    """
+else:
+    __doc__ = ""
+
+global_ναμε1 = None
+cdef double global_ναμε2 = 1.2
+
+def f():
+    """docstring"""
+    ναμε2 = 2
+    def nεsted():
+        return ναμε2
+    return nεsted
+
+# Ƒ is notably awkward because its punycode starts with "2" causing
+# C compile errors. Therefore try a few different variations...
+cdef class A:
+    cdef int ναμε
+    def __init__(self):
+        self.ναμε = 1
+    cdef Ƒ(self):
+        return self.ναμε == 1
+    def regular_function(self):
+        """
+        Can use unicode cdef functions and (private) attributes internally
+        >>> A().regular_function()
+        True
+        """
+        return self.Ƒ()
+cdef class B:
+    cpdef Ƒ(self):
+        pass
+cdef class C:
+    def Ƒ(self):
+        pass
+cdef class D:
+    cdef int Ƒ
+
+def regular_function():
+    """
+    Unicode names can be used internally on python2
+    >>> regular_function()
+    10
+    """
+    cdef int variableƑ = 5
+    ναμε2 = 2
+    return variableƑ*ναμε2
+
+cdef Fα1():
+    """docstring"""
+    ναμε2 = 2
+    raise RuntimeError() # forces generation of a traceback
+
+def print_traceback_name():
+    try:
+        Fα1()
+    except RuntimeError as e:
+        import traceback
+        # get the name of one level up in the traceback
+        print(traceback.extract_tb(e.__traceback__,2)[1][2])
+
+
+def Fα2():
+    """docstring"""
+    def nested_normal():
+        """docstring"""
+        pass
+    def nεstεd_uni():
+        """docstring"""
+        pass
+    return nested_normal, nεstεd_uni
+
+cpdef Fα3():
+    """docstring"""
+    return 1
+
+cdef class Γναμε2:
+    """
+    docstring
+    """
+    ναμε3 = 1
+
+    def __init__(self):
+        self.α = 100
+    def boring_function(self,x,ναμε5):
+        """docstring"""
+        ναμε6 = ναμε5
+        somevalue = global_ναμε1 == self.ναμε3
+        return locals()
+    def εxciting_function(self,y):
+        """docstring"""
+        def nestεd():
+            pass
+        return nestεd
+
+    cdef boring_cdef(self):
+        """docstring"""
+        pass
+    cdef εxciting_cdef(self):
+        """docstring"""
+        pass
+
+    cpdef boring_cpdef(self):
+        """docstring"""
+        return 2
+    cpdef εxciting_cpdef(self):
+        """docstring"""
+        return 2
+
+cdef class Derived(Γναμε2):
+    pass
+
+cdef Γναμε2 global_ναμε3 = Γναμε2()
+
+
+@cython.always_allow_keywords(False)  # METH_O signature
+def metho_function_taking_fancy_argument(Γναμε2 αrγ):
+    return αrγ
+
+@cython.always_allow_keywords(True)
+def function_taking_fancy_argument(Γναμε2 αrγ):
+    return αrγ
+
+
+class NormalClassΓΓ(Γναμε2):
+    """
+    docstring
+    """
+    def __init__(self):
+        self.ναμε = 10
+
+    def boring_function(self,x,ναμε5):
+        """docstring"""
+        ναμε6 = ναμε5
+        somevalue = global_ναμε1 == self.ναμε3
+        return locals()
+    def εxciting_function(self,y):
+        """docstring"""
+        def nestεd():
+            pass
+        return nestεd
+
+def unicode_kwarg(*, αrγ):
+    return αrγ
+
+def unicode_kwarg_from_cy():
+    return unicode_kwarg(αrγ=1)
+
+class NormalizeAttrPy:
+    """Python normalizes identifier names before they are used;
+    therefore fi and fi should access the same attribute"""
+    def __init__(self):
+        self.fi = 5 # note unicode ligature symbol
+    def get(self):
+        return self.fi
+
+cdef class NormalizeAttrCdef:
+    """Python normalizes identifier names before they are used;
+    therefore fi and fi should access the same attribute
+    >>> NormalizeAttrCdef().get()
+    5
+    """
+    cdef int fi # note unicode ligature symbol
+    def __init__(self):
+        self.fi = 5
+    def get(self):
+        return self.fi
+
+if sys.version_info[0]<=2:
+    # These symbols are causing problems for doctest
+    del NormalClassΓΓ
+    del globals()[u'Γναμε2'.encode('utf-8')]
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_imports.srctree cython-0.20.1+1~202203241016-9537/tests/run/unicode_imports.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_imports.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_imports.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# tag: py3, pep489
+
+PYTHON setup.py build_ext --inplace
+PYTHON -m mydoctest
+
+########### mydoctest.py #######
+
+import sys
+if sys.version_info < (3, 5):
+    # The module is only Cythonized and not build for these versions
+    # so don't run the tests
+    exit()
+    
+import doctest
+import from_py
+val = doctest.testmod(from_py)[0]
+import from_cy
+val += doctest.testmod(from_cy)[0]
+
+exit(val)
+
+########### setup.py ########
+
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import sys
+from Cython.Build import cythonize
+
+files = ["mymoð.pyx", "from_cy.pyx"]
+
+
+# For Python 2 and Python <= 3.4 just run pyx->c; 
+# don't compile the C file
+modules = cythonize(files)
+
+if sys.version_info >= (3, 5):
+    from distutils.core import setup
+
+    setup(
+        ext_modules = modules
+    )
+
+############ mymoð.pyx #########
+
+def f():
+    return True
+
+cdef public api void cdef_func():
+    pass
+
+############ pxd_moð.pxd ##########
+
+cdef struct S:
+    int x
+    
+cdef public api void cdef_func() # just to test generation of headers
+    
+############ from_py.py #########
+
+# -*- coding: utf-8 -*-
+
+import mymoð
+from mymoð import f
+
+__doc__ = """
+>>> mymoð.f()
+True
+>>> f()
+True
+"""
+
+######### from_cy.pyx ##########
+
+# -*- coding: utf-8 -*-
+
+import mymoð
+
+from mymoð import f
+
+cimport pxd_moð
+from pxd_moð cimport S
+
+
+def test_imported():
+    """
+    >>> test_imported()
+    True
+    """
+    return mymoð.f() and f() # True and True
+
+
+def test_cimported():
+    """
+    >>> test_cimported()
+    3
+    """
+    cdef pxd_moð.S v1
+    v1.x = 1
+    cdef S v2
+    v2.x = 2
+    return v1.x + v2.x
+    
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicodeliterals.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicodeliterals.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicodeliterals.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicodeliterals.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -21,6 +21,13 @@
     u'\udc00'
     >>> h
     u'\ud800'
+    >>> q
+    u'\udc00\ud800'
+
+    # The output of surrogate pairs differs between 16/32bit Unicode runtimes.
+    #>>> p
+    #u'\ud800\udc00'
+
     >>> add
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
@@ -44,6 +51,10 @@
     1
     >>> len(h)
     1
+    >>> len(q)
+    2
+    >>> len(q)
+    2
     >>> len(add)
     12
     >>> len(null)
@@ -75,19 +86,23 @@
     True
     >>> h == u'\\ud800' # unescaped by Python (required by doctest)
     True
-    >>> k == u'\\N{SNOWMAN}' == u'\\u2603'
+    >>> p == (u'\\ud800\\udc00' if sys.maxunicode == 1114111 else u'\\U00010000')  or  p  # unescaped by Python (required by doctest)
+    True
+    >>> q == u'\\udc00\\ud800'  or  q  # unescaped by Python (required by doctest)
     True
-    >>> add == u'Søk ik' + u'üÖä' + 'abc'
+    >>> k == u'\\N{SNOWMAN}' == u'\\u2603'  or  k
+    True
+    >>> m == u'abc\\\\xf8\\\\t\\u00f8\\U000000f8'  or  m  # unescaped by Python (required by doctest)
+    True
+    >>> add == u'Søk ik' + u'üÖä' + 'abc'  or  add
     True
     >>> null == u'\\x00' # unescaped by Python (required by doctest)
     True
     >>> wide_literal == u'\\U00101234'   # unescaped by Python
     True
-"""
+    >>> ustring_in_constant_tuple == ('a', u'abc', u'\\N{SNOWMAN}', u'x' * 3, u'\\N{SNOWMAN}' * 4 + u'O')  or  ustring_in_constant_tuple  # unescaped by Python
+    True
 
-if sys.version_info >= (2,6,5):
-    # this doesn't work well in older Python versions
-    __doc__ += u"""\
     >>> expected = u'\U00101234'    # unescaped by Cython
     >>> if wide_literal == expected: print(True)
     ... else: print(repr(wide_literal), repr(expected), sys.maxunicode)
@@ -110,8 +125,13 @@
 g = u'\udc00'   # lone trail surrogate
 h = u'\ud800'   # lone lead surrogate
 k = u'\N{SNOWMAN}'
+m = ur'abc\xf8\t\u00f8\U000000f8'
+p = u'\ud800\udc00'  # surrogate pair
+q = u'\udc00\ud800'  # reversed surrogate pair
 
 add = u'Søk ik' + u'üÖä' + u'abc'
 null = u'\x00'
 
 wide_literal = u'\U00101234'
+
+ustring_in_constant_tuple = ('a', u'abc', u'\N{SNOWMAN}', u'x' * 3, u'\N{SNOWMAN}' * 4 + u'O')
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicodemethods.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicodemethods.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicodemethods.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicodemethods.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -513,7 +513,9 @@
     True
     >>> mod_format(format2, ('XYZ', 'ABC')) == 'abcXYZdefABCghi'  or  mod_format(format2, ('XYZ', 'ABC'))
     True
-    >>> mod_format(None, 'sa')   # doctest: +ELLIPSIS
+
+    Exact TypeError message is different in PyPy
+    >>> mod_format(None, 'sa')   # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
     TypeError: unsupported operand type(s) for %: 'NoneType' and 'str'
     >>> class RMod(object):
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unicode_slicing.pyx cython-0.20.1+1~202203241016-9537/tests/run/unicode_slicing.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unicode_slicing.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unicode_slicing.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -151,30 +151,52 @@
     >>> slice_none_none(None, 2, 4)
     Traceback (most recent call last):    
     TypeError: 'NoneType' object is not subscriptable
+
+    >>> slice_start_end(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    
+    >>> slice_start(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    
+    >>> slice_end(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    abcdef
+    >>> slice_all(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    abcdef
+    >>> slice_start_none(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    
+    >>> slice_none_end(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    abcdef
+    >>> slice_none_none(u'abcdef', SSIZE_T_MAX, SSIZE_T_MIN)
+    abcdef
 """
 
+cdef extern from *:
+    cdef Py_ssize_t PY_SSIZE_T_MIN
+    cdef Py_ssize_t PY_SSIZE_T_MAX
+
+SSIZE_T_MAX = PY_SSIZE_T_MAX
+SSIZE_T_MIN = PY_SSIZE_T_MIN
+
 import sys
 
 if sys.version_info[0] >= 3:
     __doc__ = __doc__.replace(u"(u'", u"('").replace(u" u'", u" '")
 
-def slice_start_end(unicode s, int i, int j):
+def slice_start_end(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[i:j])
 
-def slice_start(unicode s, int i, int j):
+def slice_start(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[i:])
 
-def slice_end(unicode s, int i, int j):
+def slice_end(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[:i])
 
-def slice_all(unicode s, int i, int j):
+def slice_all(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[:])
 
-def slice_start_none(unicode s, int i, int j):
+def slice_start_none(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[i:None])
 
-def slice_none_end(unicode s, int i, int j):
+def slice_none_end(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[None:i])
 
-def slice_none_none(unicode s, int i, int j):
+def slice_none_none(unicode s, Py_ssize_t i, Py_ssize_t j):
     print(s[None:None])
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/uninitialized.py cython-0.20.1+1~202203241016-9537/tests/run/uninitialized.py
--- cython-0.20.1+1~201611251650-6686/tests/run/uninitialized.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/uninitialized.py	2022-03-24 10:16:46.000000000 +0000
@@ -5,10 +5,10 @@
     """
     >>> conditional(True)
     []
-    >>> conditional(False)
+    >>> conditional(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     if cond:
         a = []
@@ -18,10 +18,10 @@
     """
     >>> inside_loop([1,2,3])
     3
-    >>> inside_loop([])
+    >>> inside_loop([])  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'i' referenced before assignment
+    UnboundLocalError: ...local variable 'i'...
     """
     for i in iter:
         pass
@@ -31,10 +31,10 @@
     """
     >>> try_except(True)
     []
-    >>> try_except(False)
+    >>> try_except(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     try:
         if cond:
@@ -47,10 +47,10 @@
     """
     >>> try_finally(True)
     []
-    >>> try_finally(False)
+    >>> try_finally(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     try:
         if cond:
@@ -63,10 +63,10 @@
     """
     >>> deleted(False)
     {}
-    >>> deleted(True)
+    >>> deleted(True)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     a = {}
     if cond:
@@ -76,10 +76,10 @@
 def test_nested(cond):
     """
     >>> test_nested(True)
-    >>> test_nested(False)
+    >>> test_nested(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     if cond:
         def a():
@@ -90,10 +90,10 @@
     """
     >>> test_outer(True)
     {}
-    >>> test_outer(False)
+    >>> test_outer(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     if cond:
         a = {}
@@ -105,10 +105,10 @@
     """
     >>> test_inner(True)
     {}
-    >>> test_inner(False)
+    >>> test_inner(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    NameError: free variable 'a' referenced before assignment in enclosing scope
+    NameError: ...free variable 'a' ... in enclosing scope
     """
     if cond:
         a = {}
@@ -120,10 +120,10 @@
     """
     >>> test_class(True)
     1
-    >>> test_class(False)
+    >>> test_class(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'A' referenced before assignment
+    UnboundLocalError: ...local variable 'A'...
     """
     if cond:
         class A:
@@ -135,10 +135,10 @@
     """
     >>> test_try_except_regression(True)
     (123,)
-    >>> test_try_except_regression(False)
+    >>> test_try_except_regression(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     if c:
         a = (123,)
@@ -152,10 +152,10 @@
     """
     >>> test_try_finally_regression(True)
     (123,)
-    >>> test_try_finally_regression(False)
+    >>> test_try_finally_regression(False)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'a' referenced before assignment
+    UnboundLocalError: ...local variable 'a'...
     """
     if c:
         a = (123,)
@@ -169,10 +169,10 @@
     """
     >>> test_expression_calculation_order_bug(False)
     []
-    >>> test_expression_calculation_order_bug(True)
+    >>> test_expression_calculation_order_bug(True)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
     ...
-    UnboundLocalError: local variable 'b' referenced before assignment
+    UnboundLocalError: ...local variable 'b'...
     """
     if not a:
         b = []
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unpack_fused.pyx cython-0.20.1+1~202203241016-9537/tests/run/unpack_fused.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unpack_fused.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unpack_fused.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -47,18 +47,18 @@
     (1, 2)
 
     >>> items = [1, object()]
-    >>> unpack_two_int(items)
+    >>> unpack_two_int(items)  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
-    >>> unpack_two_int(iter(items))
+    TypeError: ...int...
+    >>> unpack_two_int(iter(items))  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
-    >>> unpack_two_int(list(items))
+    TypeError: ...int...
+    >>> unpack_two_int(list(items))  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
-    >>> unpack_two_int(tuple(items))
+    TypeError: ...int...
+    >>> unpack_two_int(tuple(items))  # doctest: +ELLIPSIS
     Traceback (most recent call last):
-    TypeError: an integer is required
+    TypeError: ...int...
     """
     cdef int b
     a,b = it
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unpack.pyx cython-0.20.1+1~202203241016-9537/tests/run/unpack.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unpack.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unpack.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -393,3 +393,13 @@
     cdef ExtType a,b,c,d
     a, b = c, d = None, None
     return a,b,c,d
+
+
+# Github issue #1523
+def test_unpack_resultref():
+    """
+    >>> test_unpack_resultref() == ((1, set()), 1, set())
+    True
+    """
+    a = b, c = 1, set()
+    return a, b, c
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unsignedbehaviour_T184.pyx cython-0.20.1+1~202203241016-9537/tests/run/unsignedbehaviour_T184.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unsignedbehaviour_T184.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unsignedbehaviour_T184.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 184
+# ticket: t184
 
 """
 >>> c_call()
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx cython-0.20.1+1~202203241016-9537/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 359
+# ticket: t359
 
 cdef unsigned char* some_c_unstring = 'test toast taste'
 
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/verbatiminclude.h cython-0.20.1+1~202203241016-9537/tests/run/verbatiminclude.h
--- cython-0.20.1+1~201611251650-6686/tests/run/verbatiminclude.h	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/verbatiminclude.h	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,6 @@
+static long cube(long x)
+{
+    return x * x * x;
+}
+
+#define long broken_long
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/verbatiminclude.pyx cython-0.20.1+1~202203241016-9537/tests/run/verbatiminclude.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/verbatiminclude.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/verbatiminclude.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,62 @@
+cdef extern from "verbatiminclude.h":
+    long cube(long)
+
+cdef extern from *:
+    """
+    static long square(long x)
+    {
+        return x * x;
+    }
+    """
+    long square(long)
+
+
+cdef extern from "verbatiminclude.h":
+    "typedef int myint;"
+    ctypedef int myint
+
+cdef extern from "verbatiminclude.h":
+    "#undef long"
+
+
+cdef class C:
+    cdef myint val
+
+
+cdef extern from "Python.h":
+    """
+    #define my_SET_SIZE(obj, size)  __Pyx_SET_SIZE(obj, size)
+    """
+    void my_SET_SIZE(object, Py_ssize_t)
+
+
+def test_square(x):
+    """
+    >>> test_square(4)
+    16
+    """
+    return square(x)
+
+
+def test_cube(x):
+    """
+    >>> test_cube(4)
+    64
+    """
+    return cube(x)
+
+
+def test_class():
+    """
+    >>> test_class()
+    42
+    """
+    cdef C x = C()
+    x.val = 42
+    return x.val
+
+
+def test_set_size(x, size):
+    # This function manipulates Python objects in a bad way, so we
+    # do not call it. The real test is that it compiles.
+    my_SET_SIZE(x, size)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/versioned_pxds.srctree cython-0.20.1+1~202203241016-9537/tests/run/versioned_pxds.srctree
--- cython-0.20.1+1~201611251650-6686/tests/run/versioned_pxds.srctree	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/versioned_pxds.srctree	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,79 @@
+# mode: run
+# tag: pxd
+
+"""
+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+"""
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup, Extension
+
+setup(
+  ext_modules=cythonize([
+    Extension("pkg.m1.a", ["pkg/m1/a.pyx"]),
+    Extension("pkg.m2.b", ["pkg/m2/b.pyx"])
+  ]),
+)
+
+######## pkg/__init__.py ########
+
+######## pkg/m1/__init__.py ########
+
+
+######## pkg/m1/a.pyx ########
+
+cdef class A:
+    def __init__(self):
+        self.x = 5
+
+######## pkg/m1/a.pxd ########
+
+to be ignored if there is a more specific file
+
+######## pkg/m1/a.cython-2.pxd ########
+
+very outdated, not to be picked up
+
+######## pkg/m1/a.cython-20.pxd ########
+
+outdated, not to be picked up
+
+######## pkg/m1/a.cython-29.pxd ########
+
+# closest version should get found!
+
+cdef class A:
+    cdef public float x
+
+######## pkg/m1/a.cython-300000.pxd ########
+
+Invalid distant future syntax right here!
+
+######## pkg/m1/a.cython-100000.pxd ########
+
+Invalid future syntax right here!
+
+
+######## pkg/m2/__init__.py ########
+
+######## pkg/m2/b.pyx ########
+
+from pkg.m1.a cimport A
+
+cdef class B(A):
+    pass
+
+######## runner.py ########
+
+from pkg.m1.a import A
+from pkg.m2.b import B
+
+a = A()
+b = B()
+
+assert a.x == 5
+assert isinstance(a.x, float), type(a.x)
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/with_gil_automatic.pyx cython-0.20.1+1~202203241016-9537/tests/run/with_gil_automatic.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/with_gil_automatic.pyx	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/with_gil_automatic.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,138 @@
+# mode: run
+# tag: nogil
+# cython: language_level=2
+
+cimport cython
+
+
+#### print
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//GILStatNode",
+    "//GILStatNode//GILStatNode//PrintStatNode",
+)
+def test_print_in_nogil_section(x):
+    """
+    >>> test_print_in_nogil_section(123)
+    --123--
+    """
+    with nogil:
+        print f"--{x}--"
+
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//PrintStatNode",
+)
+@cython.test_fail_if_path_exists(
+    "//GILStatNode//GILStatNode",
+)
+cpdef int test_print_in_nogil_func(x) nogil except -1:
+    """
+    >>> _ = test_print_in_nogil_func(123)
+    --123--
+    """
+    print f"--{x}--"
+
+
+#### raise
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//GILStatNode",
+    "//GILStatNode//GILStatNode//RaiseStatNode",
+)
+def test_raise_in_nogil_section(x):
+    """
+    >>> try: test_raise_in_nogil_section(123)
+    ... except ValueError as exc: print(exc)
+    ... else: print("NOT RAISED !")
+    --123--
+    """
+    with nogil:
+        raise ValueError(f"--{x}--")
+
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//RaiseStatNode",
+)
+@cython.test_fail_if_path_exists(
+    "//GILStatNode//GILStatNode",
+)
+cpdef int test_raise_in_nogil_func(x) nogil except -1:
+    """
+    >>> test_raise_in_nogil_func(123)
+    Traceback (most recent call last):
+    ValueError: --123--
+    """
+    raise ValueError(f"--{x}--")
+
+
+#### assert
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//AssertStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode",
+)
+def assert_in_nogil_section(int x):
+    """
+    >>> assert_in_nogil_section(123)
+    >>> assert_in_nogil_section(0)
+    Traceback (most recent call last):
+    AssertionError
+    """
+    with nogil:
+        assert x
+
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//AssertStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode",
+)
+def assert_in_nogil_section_ustring(int x):
+    """
+    >>> assert_in_nogil_section_string(123)
+    >>> assert_in_nogil_section_string(0)
+    Traceback (most recent call last):
+    AssertionError: failed!
+    """
+    with nogil:
+        assert x, u"failed!"
+
+
+@cython.test_assert_path_exists(
+    "//GILStatNode",
+    "//GILStatNode//AssertStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode",
+    "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode",
+)
+def assert_in_nogil_section_string(int x):
+    """
+    >>> assert_in_nogil_section_string(123)
+    >>> assert_in_nogil_section_string(0)
+    Traceback (most recent call last):
+    AssertionError: failed!
+    """
+    with nogil:
+        assert x, "failed!"
+
+
+@cython.test_assert_path_exists(
+    "//AssertStatNode",
+    "//AssertStatNode//GILStatNode",
+    "//AssertStatNode//GILStatNode//RaiseStatNode",
+)
+cpdef int assert_in_nogil_func(int x) nogil except -1:
+    """
+    >>> _ = assert_in_nogil_func(123)
+    >>> assert_in_nogil_func(0)
+    Traceback (most recent call last):
+    AssertionError: failed!
+    """
+    assert x, "failed!"
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/with_gil.pyx cython-0.20.1+1~202203241016-9537/tests/run/with_gil.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/with_gil.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/with_gil.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,3 +1,6 @@
+# mode: run
+# tag: nogil, withgil
+
 """
 Test the 'with gil:' statement.
 """
@@ -276,9 +279,11 @@
 def test_nogil_void_funcs_with_gil():
     """
     >>> redirect_stderr(test_nogil_void_funcs_with_gil)  # doctest: +ELLIPSIS
+    with_gil.ExceptionWithMsg: This is swallowed
     Exception... ignored...
     Inner gil section
     nogil section
+    ...
     Exception... ignored...
     """
     void_nogil_ignore_exception()
@@ -287,9 +292,11 @@
 def test_nogil_void_funcs_with_nogil():
     """
     >>> redirect_stderr(test_nogil_void_funcs_with_nogil)  # doctest: +ELLIPSIS
+    with_gil.ExceptionWithMsg: This is swallowed
     Exception... ignored...
     Inner gil section
     nogil section
+    with_gil.ExceptionWithMsg: Swallow this
     Exception... ignored...
     """
     with nogil:
@@ -454,3 +461,48 @@
                 with gil: print "print me first"
     except Exception, e:
         print e.args[0]
+
+
+def void_with_python_objects():
+    """
+    >>> void_with_python_objects()
+    """
+    with nogil:
+        _void_with_python_objects()
+
+
+cdef void _void_with_python_objects() nogil:
+    c = 123
+    with gil:
+        obj1 = [123]
+        obj2 = [456]
+
+
+def void_with_py_arg_reassigned(x):
+    """
+    >>> void_with_py_arg_reassigned(123)
+    """
+    with nogil:
+        _void_with_py_arg_reassigned(x)
+
+
+cdef void _void_with_py_arg_reassigned(x) nogil:
+    c = 123
+    with gil:
+        x = [456]
+
+
+cdef void test_timing_callback() with gil:
+  pass
+
+def test_timing(long N):
+  """
+  >>> sorted([test_timing(10000) for _ in range(10)])  # doctest: +ELLIPSIS
+  [...]
+  """
+  import time
+  t = time.time()
+  with nogil:
+    for _ in range(N):
+      test_timing_callback()
+  return time.time() - t
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/with_statement_module_level_T536.pyx cython-0.20.1+1~202203241016-9537/tests/run/with_statement_module_level_T536.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/with_statement_module_level_T536.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/with_statement_module_level_T536.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -1,4 +1,4 @@
-# ticket: 536
+# ticket: t536
 
 __doc__ = """
 >>> inner_result
diff -Nru cython-0.20.1+1~201611251650-6686/tests/run/yield_from_pep380.pyx cython-0.20.1+1~202203241016-9537/tests/run/yield_from_pep380.pyx
--- cython-0.20.1+1~201611251650-6686/tests/run/yield_from_pep380.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/run/yield_from_pep380.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -4,7 +4,7 @@
 Test suite for PEP 380 implementation
 
 adapted from original tests written by Greg Ewing
-see 
+see 
 """
 
 import sys
diff -Nru cython-0.20.1+1~201611251650-6686/tests/windows_bugs.txt cython-0.20.1+1~202203241016-9537/tests/windows_bugs.txt
--- cython-0.20.1+1~201611251650-6686/tests/windows_bugs.txt	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/windows_bugs.txt	2022-03-24 10:16:46.000000000 +0000
@@ -6,13 +6,27 @@
 package_compilation
 
 carray_coercion
-ctuple
 int_float_builtins_as_casts_T400
+int_float_builtins_as_casts_T400_long_double
 list_pop
 test_coroutines_pep492
 type_inference
 
 parallel
+numpy_parallel
 py_unicode_type
 
 test_grammar
+
+# Those tests don't work because MSVC wants to link to the c-algorithms library
+# when compiling and we don't want to download c-algorithms just for those tests.
+queue
+queue2
+queue3
+lunch
+
+# "C linkage function cannot return C++ class" (uses public C++ cdef function)
+cpp_template_subclasses
+
+# MSVC lacks "complex.h"
+complex_numbers_cmath_T2891
diff -Nru cython-0.20.1+1~201611251650-6686/tests/wrappers/cpp_references.pyx cython-0.20.1+1~202203241016-9537/tests/wrappers/cpp_references.pyx
--- cython-0.20.1+1~201611251650-6686/tests/wrappers/cpp_references.pyx	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/tests/wrappers/cpp_references.pyx	2022-03-24 10:16:46.000000000 +0000
@@ -5,6 +5,7 @@
 
 cdef extern from "cpp_references_helper.h":
     cdef int& ref_func(int&)
+    cdef int& except_ref_func "ref_func" (int&) except +
 
     cdef int ref_var_value
     cdef int& ref_var
@@ -29,6 +30,16 @@
     cdef int* i_ptr = &ref_func(x)
     return i_ptr[0]
 
+def test_except_ref_func_address(int x):
+    """
+    >>> test_except_ref_func_address(5)
+    5
+    >>> test_except_ref_func_address(7)
+    7
+    """
+    cdef int* i_ptr = &except_ref_func(x)
+    return i_ptr[0]
+
 def test_ref_var(int x):
     """
     >>> test_ref_func(11)
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/BUILD.bazel cython-0.20.1+1~202203241016-9537/Tools/BUILD.bazel
--- cython-0.20.1+1~201611251650-6686/Tools/BUILD.bazel	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/BUILD.bazel	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1 @@
+
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/cevaltrace.py cython-0.20.1+1~202203241016-9537/Tools/cevaltrace.py
--- cython-0.20.1+1~201611251650-6686/Tools/cevaltrace.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/cevaltrace.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+
+"""
+Translate the byte code of a Python function into the corresponding
+sequences of C code in CPython's "ceval.c".
+"""
+
+from __future__ import print_function, absolute_import
+
+import re
+import os.path
+
+from dis import get_instructions  # requires Python 3.4+
+
+# collapse some really boring byte codes
+_COLLAPSE = {'NOP', 'LOAD_CONST', 'POP_TOP', 'JUMP_FORWARD'}
+#_COLLAPSE.clear()
+
+_is_start = re.compile(r"\s* switch \s* \( opcode \)", re.VERBOSE).match
+# Py3: TARGET(XX), Py2: case XX
+_match_target = re.compile(r"\s* (?: TARGET \s* \( | case \s* ) \s* (\w+) \s* [:)]", re.VERBOSE).match
+_ignored = re.compile(r"\s* PREDICTED[A-Z_]*\(", re.VERBOSE).match
+_is_end = re.compile(r"\s* } \s* /\* \s* switch \s* \*/", re.VERBOSE).match
+
+_find_pyversion = re.compile(r'\#define \s+ PY_VERSION \s+ "([^"]+)"', re.VERBOSE).findall
+
+class ParseError(Exception):
+    def __init__(self, message="Failed to parse ceval.c"):
+        super(ParseError, self).__init__(message)
+
+
+def parse_ceval(file_path):
+    snippets = {}
+    with open(file_path) as f:
+        lines = iter(f)
+
+        for line in lines:
+            if _is_start(line):
+                break
+        else:
+            raise ParseError()
+
+        targets = []
+        code_lines = []
+        for line in lines:
+            target_match = _match_target(line)
+            if target_match:
+                if code_lines:
+                    code = ''.join(code_lines).rstrip()
+                    for target in targets:
+                        snippets[target] = code
+                    del code_lines[:], targets[:]
+                targets.append(target_match.group(1))
+            elif _ignored(line):
+                pass
+            elif _is_end(line):
+                break
+            else:
+                code_lines.append(line)
+        else:
+            if not snippets:
+                raise ParseError()
+    return snippets
+
+
+def translate(func, ceval_snippets):
+    start_offset = 0
+    code_obj = getattr(func, '__code__', None)
+    if code_obj and os.path.exists(code_obj.co_filename):
+        start_offset = code_obj.co_firstlineno
+        with open(code_obj.co_filename) as f:
+            code_line_at = {
+                i: line.strip()
+                for i, line in enumerate(f, 1)
+                if line.strip()
+            }.get
+    else:
+        code_line_at = lambda _: None
+
+    for instr in get_instructions(func):
+        code_line = code_line_at(instr.starts_line)
+        line_no = (instr.starts_line or start_offset) - start_offset
+        yield line_no, code_line, instr, ceval_snippets.get(instr.opname)
+
+
+def main():
+    import sys
+    import importlib.util
+
+    if len(sys.argv) < 3:
+        print("Usage:  %s  path/to/Python/ceval.c  script.py ..." % sys.argv[0], file=sys.stderr)
+        return
+
+    ceval_source_file = sys.argv[1]
+    version_header = os.path.join(os.path.dirname(ceval_source_file), '..', 'Include', 'patchlevel.h')
+    if os.path.exists(version_header):
+        with open(version_header) as f:
+            py_version = _find_pyversion(f.read())
+        if py_version:
+            py_version = py_version[0]
+            if not sys.version.startswith(py_version + ' '):
+                print("Warning:  disassembling with Python %s, but ceval.c has version %s" % (
+                    sys.version.split(None, 1)[0],
+                    py_version,
+                ), file=sys.stderr)
+
+    snippets = parse_ceval(ceval_source_file)
+
+    for code in _COLLAPSE:
+        if code in snippets:
+            snippets[code] = ''
+
+    for file_path in sys.argv[2:]:
+        module_name = os.path.basename(file_path)
+        print("/*######## MODULE %s ########*/" % module_name)
+        print('')
+
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        for func_name, item in sorted(vars(module).items()):
+            if not callable(item):
+                continue
+            print("/* FUNCTION %s */" % func_name)
+            print("static void")  # assuming that it highlights in editors
+            print("%s() {" % func_name)
+
+            last_line = None
+            for line_no, code_line, instr, snippet in translate(item, snippets):
+                if last_line != line_no:
+                    if code_line:
+                        print('')
+                        print('/*# %3d  %s */' % (line_no, code_line))
+                        print('')
+                    last_line = line_no
+
+                print("  %s:%s {%s" % (
+                    instr.opname,
+                    ' /* %s */' % instr.argrepr if instr.arg is not None else '',
+                    ' /* ??? */' if snippet is None else ' /* ... */ }' if snippet == '' else '',
+                ))
+                print(snippet or '')
+
+            print("} /* FUNCTION %s */" % func_name)
+
+
+if __name__ == '__main__':
+    main()
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/ci-run.sh cython-0.20.1+1~202203241016-9537/Tools/ci-run.sh
--- cython-0.20.1+1~201611251650-6686/Tools/ci-run.sh	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/ci-run.sh	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,176 @@
+#!/usr/bin/bash
+
+GCC_VERSION=${GCC_VERSION:=8}
+
+# Set up compilers
+if [[ $TEST_CODE_STYLE == "1" ]]; then
+  echo "Skipping compiler setup"
+elif [[ $OSTYPE == "linux-gnu"* ]]; then
+  echo "Setting up linux compiler"
+  echo "Installing requirements [apt]"
+  sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
+  sudo apt update -y -q
+  sudo apt install -y -q ccache gdb python-dbg python3-dbg gcc-$GCC_VERSION || exit 1
+
+  ALTERNATIVE_ARGS=""
+  if [[ $BACKEND == *"cpp"* ]]; then
+    sudo apt install -y -q g++-$GCC_VERSION || exit 1
+    ALTERNATIVE_ARGS="--slave /usr/bin/g++ g++ /usr/bin/g++-$GCC_VERSION"
+  fi
+  sudo /usr/sbin/update-ccache-symlinks
+  echo "/usr/lib/ccache" >> $GITHUB_PATH # export ccache to path
+
+  sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GCC_VERSION 60 $ALTERNATIVE_ARGS
+
+  export CC="gcc"
+  if [[ $BACKEND == *"cpp"* ]]; then
+    sudo update-alternatives --set g++ /usr/bin/g++-$GCC_VERSION
+    export CXX="g++"
+  fi
+elif [[ $OSTYPE == "darwin"* ]]; then
+  echo "Setting up macos compiler"
+  export CC="clang -Wno-deprecated-declarations"
+  export CXX="clang++ -stdlib=libc++ -Wno-deprecated-declarations"
+else
+  echo "No setup specified for $OSTYPE"
+fi
+
+# Set up miniconda
+if [[ $STACKLESS == "true" ]]; then
+  echo "Installing stackless python"
+  #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt
+  conda config --add channels stackless
+  conda install --quiet --yes stackless || exit 1
+fi
+
+PYTHON_SYS_VERSION=$(python -c 'import sys; print(sys.version)')
+
+# Log versions in use
+echo "===================="
+echo "|VERSIONS INSTALLED|"
+echo "===================="
+echo "Python $PYTHON_SYS_VERSION"
+if [[ $CC ]]; then
+  which ${CC%% *}
+  ${CC%% *} --version
+fi
+if [[ $CXX ]]; then
+  which ${CXX%% *}
+  ${CXX%% *} --version
+fi
+echo "===================="
+
+# Install python requirements
+echo "Installing requirements [python]"
+if [[ $PYTHON_VERSION == "2.7"* ]]; then
+  pip install wheel || exit 1
+  pip install -r test-requirements-27.txt || exit 1
+elif [[ $PYTHON_VERSION == "3."[45]* ]]; then
+  python -m pip install wheel || exit 1
+  python -m pip install -r test-requirements-34.txt || exit 1
+else
+  python -m pip install -U pip "setuptools<60" wheel || exit 1
+
+  if [[ $PYTHON_VERSION != *"-dev" || $COVERAGE == "1" ]]; then
+    python -m pip install -r test-requirements.txt || exit 1
+
+    if [[ $PYTHON_VERSION != "pypy"* && $PYTHON_VERSION != "3."[1]* ]]; then
+      python -m pip install -r test-requirements-cpython.txt || exit 1
+    fi
+  fi
+fi
+
+if [[ $TEST_CODE_STYLE == "1" ]]; then
+  STYLE_ARGS="--no-unit --no-doctest --no-file --no-pyregr --no-examples"
+  python -m pip install -r doc-requirements.txt || exit 1
+else
+  STYLE_ARGS="--no-code-style"
+
+  # Install more requirements
+  if [[ $PYTHON_VERSION != *"-dev" ]]; then
+    if [[ $BACKEND == *"cpp"* ]]; then
+      echo "WARNING: Currently not installing pythran due to compatibility issues"
+      # python -m pip install pythran==0.9.5 || exit 1
+    fi
+
+    if [[ $BACKEND != "cpp" && $PYTHON_VERSION != "pypy"* &&
+          $PYTHON_VERSION != "2"* && $PYTHON_VERSION != "3.4"* ]]; then
+      python -m pip install mypy || exit 1
+    fi
+  fi
+fi
+
+# Run tests
+echo "==== Running tests ===="
+ccache -s 2>/dev/null || true
+export PATH="/usr/lib/ccache:$PATH"
+
+# Most modern compilers allow the last conflicting option
+# to override the previous ones, so '-O0 -O3' == '-O3'
+# This is true for the latest msvc, gcc and clang
+CFLAGS="-O0 -ggdb -Wall -Wextra"
+
+if [[ $NO_CYTHON_COMPILE != "1" && $PYTHON_VERSION != "pypy"* ]]; then
+
+  BUILD_CFLAGS="$CFLAGS -O2"
+  if [[ $PYTHON_SYS_VERSION == "2"* ]]; then
+    BUILD_CFLAGS="$BUILD_CFLAGS -fno-strict-aliasing"
+  fi
+
+  SETUP_ARGS=""
+  if [[ $COVERAGE == "1" ]]; then
+    SETUP_ARGS="$SETUP_ARGS --cython-coverage"
+  fi
+  if [[ $CYTHON_COMPILE_ALL == "1" ]]; then
+    SETUP_ARGS="$SETUP_ARGS --cython-compile-all"
+  fi
+  SETUP_ARGS="$SETUP_ARGS
+    $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")')"
+
+  CFLAGS=$BUILD_CFLAGS \
+    python setup.py build_ext -i $SETUP_ARGS || exit 1
+
+  # COVERAGE can be either "" (empty or not set) or "1" (when we set it)
+  # STACKLESS can be either  "" (empty or not set) or "true" (when we set it)
+  # CYTHON_COMPILE_ALL can be either  "" (empty or not set) or "1" (when we set it)
+  if [[ $COVERAGE != "1" && $STACKLESS != "true" && $BACKEND != *"cpp"* &&
+        $CYTHON_COMPILE_ALL != "1" && $LIMITED_API == "" && $EXTRA_CFLAGS == "" ]]; then
+    python setup.py bdist_wheel || exit 1
+  fi
+fi
+
+if [[ $TEST_CODE_STYLE == "1" ]]; then
+  make -C docs html || exit 1
+elif [[ $PYTHON_VERSION != "pypy"* ]]; then
+  # Run the debugger tests in python-dbg if available
+  # (but don't fail, because they currently do fail)
+  PYTHON_DBG=$(python -c 'import sys; print("%d.%d" % sys.version_info[:2])')
+  PYTHON_DBG="python$PYTHON_DBG-dbg"
+  if $PYTHON_DBG -V >&2; then
+    CFLAGS=$CFLAGS $PYTHON_DBG \
+      runtests.py -vv --no-code-style Debugger --backends=$BACKEND
+  fi
+fi
+
+RUNTESTS_ARGS=""
+if [[ $COVERAGE == "1" ]]; then
+  RUNTESTS_ARGS="$RUNTESTS_ARGS --coverage --coverage-html --cython-only"
+fi
+if [[ $TEST_CODE_STYLE != "1" ]]; then
+  RUNTESTS_ARGS="$RUNTESTS_ARGS -j7"
+fi
+
+export CFLAGS="$CFLAGS $EXTRA_CFLAGS"
+python runtests.py \
+  -vv $STYLE_ARGS \
+  -x Debugger \
+  --backends=$BACKEND \
+  $LIMITED_API \
+  $EXCLUDE \
+  $RUNTESTS_ARGS
+
+EXIT_CODE=$?
+
+ccache -s 2>/dev/null || true
+
+exit $EXIT_CODE
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/cython-mode.el cython-0.20.1+1~202203241016-9537/Tools/cython-mode.el
--- cython-0.20.1+1~201611251650-6686/Tools/cython-mode.el	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/cython-mode.el	2022-03-24 10:16:46.000000000 +0000
@@ -1,5 +1,7 @@
 ;;; cython-mode.el --- Major mode for editing Cython files
 
+;; License: Apache-2.0
+
 ;;; Commentary:
 
 ;; This should work with python-mode.el as well as either the new
@@ -52,7 +54,7 @@
        symbol-start (group "ctypedef")
        ;; type specifier: at least 1 non-identifier symbol + 1 identifier
        ;; symbol and anything but a comment-starter after that.
-       (opt (regexp "[^a-zA-z0-9_\n]+[a-zA-Z0-9_][^#\n]*")
+       (opt (regexp "[^a-zA-Z0-9_\n]+[a-zA-Z0-9_][^#\n]*")
             ;; type alias: an identifier
             symbol-start (group (regexp "[a-zA-Z_]+[a-zA-Z0-9_]*"))
             ;; space-or-comments till the end of the line
@@ -101,7 +103,7 @@
 (defgroup cython nil "Major mode for editing and compiling Cython files"
   :group 'languages
   :prefix "cython-"
-  :link '(url-link :tag "Homepage" "http://cython.org"))
+  :link '(url-link :tag "Homepage" "https://cython.org/"))
 
 ;;;###autoload
 (defcustom cython-default-compile-format "cython -a %s"
@@ -262,16 +264,25 @@
   (save-excursion
     ;; Move up the tree of nested `class' and `def' blocks until we
     ;; get to zero indentation, accumulating the defined names.
-    (let ((start t)
+    (let ((not-finished t)
           accum)
-      (while (or start (> (current-indentation) 0))
-        (setq start nil)
-        (cython-beginning-of-block)
-        (end-of-line)
-        (beginning-of-defun)
-        (if (looking-at (rx (0+ space) (or "def" "cdef" "cpdef" "class") (1+ space)
+      (skip-chars-backward " \t\r\n")
+      (cython-beginning-of-defun)
+      (while not-finished
+	(beginning-of-line)
+	(skip-chars-forward " \t\r\n")
+        (if (looking-at (rx (0+ space) (or "def" "class") (1+ space)
                             (group (1+ (or word (syntax symbol))))))
-            (push (match-string 1) accum)))
+	    (push (match-string 1) accum)
+	  (if (looking-at (rx (0+ space) (or "cdef" "cpdef") (1+ space) (1+ word) (1+ space)
+                              (group (1+ (or word (syntax symbol))))))
+	      (push (match-string 1) accum)))
+	(let ((indentation (current-indentation)))
+	  (if (= 0 indentation)
+	      (setq not-finished nil)
+	    (while (= indentation (current-indentation))
+	      (message "%s %s" indentation (current-indentation))
+	      (cython-beginning-of-defun)))))
       (if accum (mapconcat 'identity accum ".")))))
 
 ;;;###autoload
@@ -289,7 +300,7 @@
   (set (make-local-variable 'end-of-defun-function)
        #'cython-end-of-defun)
   (set (make-local-variable 'compile-command)
-       (format cython-default-compile-format (shell-quote-argument buffer-file-name)))
+       (format cython-default-compile-format (shell-quote-argument (or buffer-file-name ""))))
   (set (make-local-variable 'add-log-current-defun-function)
        #'cython-current-defun)
   (add-hook 'which-func-functions #'cython-current-defun nil t)
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/cython-numpy-mode-kate.xml cython-0.20.1+1~202203241016-9537/Tools/cython-numpy-mode-kate.xml
--- cython-0.20.1+1~201611251650-6686/Tools/cython-numpy-mode-kate.xml	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/cython-numpy-mode-kate.xml	2022-03-24 10:16:46.000000000 +0000
@@ -9,14 +9,14 @@
 
 
@@ -216,7 +216,7 @@
         
         
              cython 
-                
+        
         
               dtype 
               flatiter 
@@ -244,48 +244,48 @@
               cfloat_t 
               cdouble_t 
               clongdouble_t 
-              complex_t             
-              npy_int8  
-              npy_int16  
-              npy_int32  
-              npy_int64  
-              npy_int96  
-              npy_int128  
-              npy_uint8  
-              npy_uint16  
-              npy_uint32  
-              npy_uint64  
-              npy_uint96  
-              npy_uint128  
-              npy_float32  
-              npy_float64  
-              npy_float80  
-              npy_float96  
-              npy_float128  
-              npy_complex64  
-              npy_complex128  
-              npy_complex120  
-              npy_complex192  
-              npy_complex256  
-              npy_cfloat  
-              npy_cdouble  
-              npy_clongdouble  
-              npy_bool  
-              npy_byte  
-              npy_short  
-              npy_int  
-              npy_long  
-              npy_longlong  
-              npy_ubyte  
-              npy_ushort  
-              npy_uint  
-              npy_ulong  
-              npy_ulonglong  
-              npy_float  
-              npy_double  
-              npy_longdouble  
+              complex_t 
+              npy_int8 
+              npy_int16 
+              npy_int32 
+              npy_int64 
+              npy_int96 
+              npy_int128 
+              npy_uint8 
+              npy_uint16 
+              npy_uint32 
+              npy_uint64 
+              npy_uint96 
+              npy_uint128 
+              npy_float32 
+              npy_float64 
+              npy_float80 
+              npy_float96 
+              npy_float128 
+              npy_complex64 
+              npy_complex128 
+              npy_complex120 
+              npy_complex192 
+              npy_complex256 
+              npy_cfloat 
+              npy_cdouble 
+              npy_clongdouble 
+              npy_bool 
+              npy_byte 
+              npy_short 
+              npy_int 
+              npy_long 
+              npy_longlong 
+              npy_ubyte 
+              npy_ushort 
+              npy_uint 
+              npy_ulong 
+              npy_ulonglong 
+              npy_float 
+              npy_double 
+              npy_longdouble 
               npy_intp 
-   		        
+   		
         
              DataSource 
              MachAr 
@@ -868,43 +868,43 @@
             WindowsError
             ZeroDivisionError
         
-        
+
         
-                      
+
 			
-                
+
                 
                 
                 
-                
+
 				
                 
 				
 				
 				
 				
-                
+
 				
 				
                 
-                
+
                 
 				
-                                                                                
+
                 
                 
                 
-                
+
 				
 				
-				
+
                 
 				
-				
+
                 
-                
+
                 
-                
+
                 
 
 				
@@ -916,115 +916,115 @@
 				
 				
 				
-       
+
   				
 				
 				
 				
-                
-                
+
+
                 
-				
+
                 
-                
+
                 
-                              
+
 				
-                
+
 				
-			
+
             
-            
+
              
-                 
+                
                 
                 
                 
                 
             
-                                    
+
 			
-                 
+                
 				
 			
-            
+
             
-                   
-                 
+                
+                
                 
             
-            
+
             
                 
                 
             
-            
+
             
                 
                 
                 
                 
             
-            
+
             
                 
                 
             
-            
+
             
                 
                 
             
-            
-            
+
+
             
                 
                 
                 
                 
                 
-                
+
                 
                 
-                                                                                
+
                 
                 
                 
-                
+
                 
                 
-                
+
                 
                 
-                
+
                 
-                                
+
                 
                 
                 
             
-            
+
             
                 
                 
             
-            
-            
-            
-            
-            
-            
-            
-            
+
+
+
+
+
+
+
+
             
                 
                 
                 
             
-            
-            
-            
-            
+
+
+
+
 			
 				
 			
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/dump_github_issues.py cython-0.20.1+1~202203241016-9537/Tools/dump_github_issues.py
--- cython-0.20.1+1~201611251650-6686/Tools/dump_github_issues.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/dump_github_issues.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,142 @@
+"""
+Dump the GitHub issues of the current project to a file (.json.gz).
+
+Usage:  python3 Tools/dump_github_issues.py
+"""
+
+import configparser
+import gzip
+import json
+import os.path
+
+from datetime import datetime
+from urllib.request import urlopen
+
+GIT_CONFIG_FILE = ".git/config"
+
+
+class RateLimitReached(Exception):
+    pass
+
+
+def gen_urls(repo):
+    i = 0
+    while True:
+        yield f"https://api.github.com/repos/{repo}/issues?state=all&per_page=100&page={i}"
+        i += 1
+
+
+def read_rate_limit():
+    with urlopen("https://api.github.com/rate_limit") as p:
+        return json.load(p)
+
+
+def parse_rate_limit(limits):
+    limits = limits['resources']['core']
+    return limits['limit'], limits['remaining'], datetime.fromtimestamp(limits['reset'])
+
+
+def load_url(url):
+    with urlopen(url) as p:
+        data = json.load(p)
+    if isinstance(data, dict) and 'rate limit' in data.get('message', ''):
+        raise RateLimitReached()
+
+    assert isinstance(data, list), type(data)
+    return data or None  # None indicates empty last page
+
+
+def join_list_data(lists):
+    result = []
+    for data in lists:
+        if not data:
+            break
+        result.extend(data)
+    return result
+
+
+def output_filename(repo):
+    timestamp = datetime.now()
+    return f"github_issues_{repo.replace('/', '_')}_{timestamp.strftime('%Y%m%d_%H%M%S')}.json.gz"
+
+
+def write_gzjson(file_name, data, indent=2):
+    with gzip.open(file_name, "wt", encoding='utf-8') as gz:
+        json.dump(data, gz, indent=indent)
+
+
+def find_origin_url(git_config=GIT_CONFIG_FILE):
+    assert os.path.exists(git_config)
+    parser = configparser.ConfigParser()
+    parser.read(git_config)
+    return parser.get('remote "origin"', 'url')
+
+
+def parse_repo_name(git_url):
+    if git_url.endswith('.git'):
+        git_url = git_url[:-4]
+    return '/'.join(git_url.split('/')[-2:])
+
+
+def dump_issues(repo):
+    """Main entry point."""
+    print(f"Reading issues from repo '{repo}'")
+    urls = gen_urls(repo)
+    try:
+        paged_data = map(load_url, urls)
+        issues = join_list_data(paged_data)
+    except RateLimitReached:
+        limit, remaining, reset_time = parse_rate_limit(read_rate_limit())
+        print(f"FAILURE: Rate limits ({limit}) reached, remaining: {remaining}, reset at {reset_time}")
+        return
+
+    filename = output_filename(repo)
+    print(f"Writing {len(issues)} to {filename}")
+    write_gzjson(filename, issues)
+
+
+### TESTS
+
+def test_join_list_data():
+    assert join_list_data([]) == []
+    assert join_list_data([[1,2]]) == [1,2]
+    assert join_list_data([[1,2], [3]]) == [1,2,3]
+    assert join_list_data([[0], [1,2], [3]]) == [0,1,2,3]
+    assert join_list_data([[0], [1,2], [[[]],[]]]) == [0,1,2,[[]],[]]
+
+
+def test_output_filename():
+    filename = output_filename("re/po")
+    import re
+    assert re.match(r"github_issues_re_po_[0-9]{8}_[0-9]{6}\.json", filename)
+
+
+def test_find_origin_url():
+    assert find_origin_url()
+
+
+def test_parse_repo_name():
+    assert parse_repo_name("https://github.com/cython/cython") == "cython/cython"
+    assert parse_repo_name("git+ssh://git@github.com/cython/cython.git") == "cython/cython"
+    assert parse_repo_name("git+ssh://git@github.com/fork/cython.git") == "fork/cython"
+
+
+def test_write_gzjson():
+    import tempfile
+    with tempfile.NamedTemporaryFile() as tmp:
+        write_gzjson(tmp.name, [{}])
+
+        # test JSON format
+        with gzip.open(tmp.name) as f:
+            assert json.load(f) == [{}]
+
+        # test indentation
+        with gzip.open(tmp.name) as f:
+            assert f.read() == b'[\n  {}\n]'
+
+
+### MAIN
+
+if __name__ == '__main__':
+    repo_name = parse_repo_name(find_origin_url())
+    dump_issues(repo_name)
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/gen_tests_for_posix_pxds.py cython-0.20.1+1~202203241016-9537/Tools/gen_tests_for_posix_pxds.py
--- cython-0.20.1+1~201611251650-6686/Tools/gen_tests_for_posix_pxds.py	1970-01-01 00:00:00.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/gen_tests_for_posix_pxds.py	2022-03-24 10:16:46.000000000 +0000
@@ -0,0 +1,41 @@
+#!/usr/bin/python3
+
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__) / "../.."
+POSIX_PXDS_DIR = PROJECT_ROOT / "Cython/Includes/posix"
+TEST_PATH = PROJECT_ROOT / "tests/compile/posix_pxds.pyx"
+
+def main():
+    datas = [
+        "# tag: posix\n"
+        "# mode: compile\n"
+        "\n"
+        "# This file is generated by `Tools/gen_tests_for_posix_pxds.py`.\n"
+        "\n"
+        "cimport posix\n"
+    ]
+
+    filenames = sorted(map(lambda path: path.name, POSIX_PXDS_DIR.iterdir()))
+
+    for name in filenames:
+        if name == "__init__.pxd":
+            continue
+        if name.endswith(".pxd"):
+            name = name[:-4]
+        else:
+            continue
+
+        s = (
+            "cimport posix.{name}\n"
+            "from posix cimport {name}\n"
+            "from posix.{name} cimport *\n"
+        ).format(name=name)
+
+        datas.append(s)
+
+    with open(TEST_PATH, "w", encoding="utf-8", newline="\n") as f:
+        f.write("\n".join(datas))
+
+if __name__ == "__main__":
+    main()
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/jedityper.py cython-0.20.1+1~202203241016-9537/Tools/jedityper.py
--- cython-0.20.1+1~201611251650-6686/Tools/jedityper.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/jedityper.py	2022-03-24 10:16:46.000000000 +0000
@@ -31,7 +31,7 @@
         raise ValueError("Either 'source_path' or 'code' is required.")
     scoped_names = {}
     statement_iter = jedi.names(source=code, path=source_path, all_scopes=True)
-    
+
     for statement in statement_iter:
         parent = statement.parent()
         scope = parent._definition
@@ -45,7 +45,7 @@
             names = scoped_names[key]
         except KeyError:
             names = scoped_names[key] = defaultdict(set)
-                
+
         position = statement.start_pos if statement.name in names else None
 
         for name_type in evaluator.find_types(scope, statement.name, position=position ,search_global=True):
@@ -60,8 +60,8 @@
                 type_name = None
             else:
                 try:
-                    type_name = type(name_type.obj).__name__                
-                except AttributeError as error:                    
+                    type_name = type(name_type.obj).__name__
+                except AttributeError as error:
                     type_name = None
             if type_name is not None:
                 names[str(statement.name)].add(type_name)
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/kate.diff cython-0.20.1+1~202203241016-9537/Tools/kate.diff
--- cython-0.20.1+1~201611251650-6686/Tools/kate.diff	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/kate.diff	2022-03-24 10:16:46.000000000 +0000
@@ -20,14 +20,14 @@
 +
 +
@@ -227,7 +227,7 @@
 +        
 +        
 +             cython 
-+                
++        
 +        
 +              dtype 
 +              flatiter 
@@ -255,48 +255,48 @@
 +              cfloat_t 
 +              cdouble_t 
 +              clongdouble_t 
-+              complex_t             
-+              npy_int8  
-+              npy_int16  
-+              npy_int32  
-+              npy_int64  
-+              npy_int96  
-+              npy_int128  
-+              npy_uint8  
-+              npy_uint16  
-+              npy_uint32  
-+              npy_uint64  
-+              npy_uint96  
-+              npy_uint128  
-+              npy_float32  
-+              npy_float64  
-+              npy_float80  
-+              npy_float96  
-+              npy_float128  
-+              npy_complex64  
-+              npy_complex128  
-+              npy_complex120  
-+              npy_complex192  
-+              npy_complex256  
-+              npy_cfloat  
-+              npy_cdouble  
-+              npy_clongdouble  
-+              npy_bool  
-+              npy_byte  
-+              npy_short  
-+              npy_int  
-+              npy_long  
-+              npy_longlong  
-+              npy_ubyte  
-+              npy_ushort  
-+              npy_uint  
-+              npy_ulong  
-+              npy_ulonglong  
-+              npy_float  
-+              npy_double  
-+              npy_longdouble  
++              complex_t 
++              npy_int8 
++              npy_int16 
++              npy_int32 
++              npy_int64 
++              npy_int96 
++              npy_int128 
++              npy_uint8 
++              npy_uint16 
++              npy_uint32 
++              npy_uint64 
++              npy_uint96 
++              npy_uint128 
++              npy_float32 
++              npy_float64 
++              npy_float80 
++              npy_float96 
++              npy_float128 
++              npy_complex64 
++              npy_complex128 
++              npy_complex120 
++              npy_complex192 
++              npy_complex256 
++              npy_cfloat 
++              npy_cdouble 
++              npy_clongdouble 
++              npy_bool 
++              npy_byte 
++              npy_short 
++              npy_int 
++              npy_long 
++              npy_longlong 
++              npy_ubyte 
++              npy_ushort 
++              npy_uint 
++              npy_ulong 
++              npy_ulonglong 
++              npy_float 
++              npy_double 
++              npy_longdouble 
 +              npy_intp 
-+   		        
++   		
 +        
 +             DataSource 
 +             MachAr 
@@ -879,43 +879,43 @@
 +            WindowsError
 +            ZeroDivisionError
 +        
-+        
++
 +        
-+                      
++
 +			
-+                
++
 +                
 +                
 +                
-+                
++
 +				
 +                
 +				
 +				
 +				
 +				
-+                
++
 +				
 +				
 +                
-+                
++
 +                
 +				
-+                                                                                
++
 +                
 +                
 +                
-+                
++
 +				
 +				
-+				
++
 +                
 +				
-+				
++
 +                
-+                
++
 +                
-+                
++
 +                
 +
 +				
@@ -927,115 +927,115 @@
 +				
 +				
 +				
-+       
++
 +  				
 +				
 +				
 +				
-+                
-+                
++
++
 +                
-+				
++
 +                
-+                
++
 +                
-+                              
++
 +				
-+                
++
 +				
-+			
++
 +            
-+            
++
 +             
-+                 
++                
 +                
 +                
 +                
 +                
 +            
-+                                    
++
 +			
-+                 
++                
 +				
 +			
-+            
++
 +            
-+                   
-+                 
++                
++                
 +                
 +            
-+            
++
 +            
 +                
 +                
 +            
-+            
++
 +            
 +                
 +                
 +                
 +                
 +            
-+            
++
 +            
 +                
 +                
 +            
-+            
++
 +            
 +                
 +                
 +            
-+            
-+            
++
++
 +            
 +                
 +                
 +                
 +                
 +                
-+                
++
 +                
 +                
-+                                                                                
++
 +                
 +                
 +                
-+                
++
 +                
 +                
-+                
++
 +                
 +                
-+                
++
 +                
-+                                
++
 +                
 +                
 +                
 +            
-+            
++
 +            
 +                
 +                
 +            
-+            
-+            
-+            
-+            
-+            
-+            
-+            
-+            
++
++
++
++
++
++
++
++
 +            
 +                
 +                
 +                
 +            
-+            
-+            
-+            
-+            
++
++
++
++
 +			
 +				
 +			
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/rules.bzl cython-0.20.1+1~202203241016-9537/Tools/rules.bzl
--- cython-0.20.1+1~201611251650-6686/Tools/rules.bzl	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/rules.bzl	2022-03-24 10:16:46.000000000 +0000
@@ -11,8 +11,8 @@
 
 pyx_library(name = 'mylib',
             srcs = ['a.pyx', 'a.pxd', 'b.py', 'pkg/__init__.py', 'pkg/c.pyx'],
-            py_deps = ['//py_library/dep'],
-            data = ['//other/data'],
+            # python library deps passed to py_library
+            deps = ['//py_library/dep']
 )
 
 The __init__.py file must be in your srcs list so that Cython can resolve
@@ -24,13 +24,14 @@
         deps=[],
         srcs=[],
         cython_directives=[],
-        cython_options=[]):
+        cython_options=[],
+        **kwargs):
     # First filter out files that should be run compiled vs. passed through.
     py_srcs = []
     pyx_srcs = []
     pxd_srcs = []
     for src in srcs:
-        if src.endswith('.pyx') or (src.endwith('.py')
+        if src.endswith('.pyx') or (src.endswith('.py')
                                     and src[:-3] + '.pxd' in srcs):
             pyx_srcs.append(src)
         elif src.endswith('.py'):
@@ -47,7 +48,7 @@
                            ["-s '%s=%s'" % x for x in cython_options])
     # TODO(robertwb): It might be better to only generate the C files,
     # letting cc_library (or similar) handle the rest, but there isn't yet
-    # suport compiling Python C extensions from bazel.
+    # support compiling Python C extensions from bazel.
     native.genrule(
         name = name + "_cythonize",
         srcs = pyx_srcs,
@@ -64,5 +65,6 @@
         name=name,
         srcs=py_srcs,
         deps=deps,
-        data=outs + pyx_srcs + pxd_srcs
+        data=outs + pyx_srcs + pxd_srcs,
+        **kwargs
     )
diff -Nru cython-0.20.1+1~201611251650-6686/Tools/site_scons/site_tools/pyext.py cython-0.20.1+1~202203241016-9537/Tools/site_scons/site_tools/pyext.py
--- cython-0.20.1+1~201611251650-6686/Tools/site_scons/site_tools/pyext.py	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/Tools/site_scons/site_tools/pyext.py	2022-03-24 10:16:46.000000000 +0000
@@ -116,7 +116,7 @@
     return pyext_cccom, pyext_cxxcom, pyext_linkcom
 
 def set_basic_vars(env):
-    # Set construction variables which are independant on whether we are using
+    # Set construction variables which are independent on whether we are using
     # distutils or not.
     env['PYEXTCPPPATH'] = SCons.Util.CLVar('$PYEXTINCPATH')
 
@@ -133,7 +133,7 @@
     # this work, we need to know whether PYEXTCC accepts /c and /Fo or -c -o.
     # This is difficult with the current way tools work in scons.
     pycc, pycxx, pylink = pyext_coms(sys.platform)
-                            
+
     env['PYEXTLINKFLAGSEND'] = SCons.Util.CLVar('$LINKFLAGSEND')
 
     env['PYEXTCCCOM'] = pycc
@@ -157,7 +157,7 @@
     for k, v in def_cfg.items():
         ifnotset(env, k, v)
 
-    ifnotset(env, 'PYEXT_ALLOW_UNDEFINED', 
+    ifnotset(env, 'PYEXT_ALLOW_UNDEFINED',
              SCons.Util.CLVar('$ALLOW_UNDEFINED'))
     ifnotset(env, 'PYEXTLINKFLAGS', SCons.Util.CLVar('$LDMODULEFLAGS'))
 
@@ -178,12 +178,12 @@
     # We define commands as strings so that we can either execute them using
     # eval (same python for scons and distutils) or by executing them through
     # the shell.
-    dist_cfg = {'PYEXTCC': ("sysconfig.get_config_var('CC')", False), 
-                'PYEXTCFLAGS': ("sysconfig.get_config_var('CFLAGS')", True), 
-                'PYEXTCCSHARED': ("sysconfig.get_config_var('CCSHARED')", False), 
-                'PYEXTLINKFLAGS': ("sysconfig.get_config_var('LDFLAGS')", True), 
-                'PYEXTLINK': ("sysconfig.get_config_var('LDSHARED')", False), 
-                'PYEXTINCPATH': ("sysconfig.get_python_inc()", False), 
+    dist_cfg = {'PYEXTCC': ("sysconfig.get_config_var('CC')", False),
+                'PYEXTCFLAGS': ("sysconfig.get_config_var('CFLAGS')", True),
+                'PYEXTCCSHARED': ("sysconfig.get_config_var('CCSHARED')", False),
+                'PYEXTLINKFLAGS': ("sysconfig.get_config_var('LDFLAGS')", True),
+                'PYEXTLINK': ("sysconfig.get_config_var('LDSHARED')", False),
+                'PYEXTINCPATH': ("sysconfig.get_python_inc()", False),
                 'PYEXTSUFFIX': ("sysconfig.get_config_var('SO')", False)}
 
     from distutils import sysconfig
@@ -208,7 +208,7 @@
     if 'PYEXT_USE_DISTUTILS' not in env:
         env['PYEXT_USE_DISTUTILS'] = False
 
-    # This sets all constructions variables used for pyext builders. 
+    # This sets all constructions variables used for pyext builders.
     set_basic_vars(env)
 
     set_configuration(env, env['PYEXT_USE_DISTUTILS'])
diff -Nru cython-0.20.1+1~201611251650-6686/.travis.yml cython-0.20.1+1~202203241016-9537/.travis.yml
--- cython-0.20.1+1~201611251650-6686/.travis.yml	2016-11-25 16:50:54.000000000 +0000
+++ cython-0.20.1+1~202203241016-9537/.travis.yml	2022-03-24 10:16:46.000000000 +0000
@@ -1,64 +1,48 @@
+os: linux
 language: python
 
-sudo: false
+addons:
+  apt:
+    packages:
+      - gdb
+      - python-dbg
+      - python3-dbg
+      - libzmq-dev  # needed by IPython/Tornado
+      #- gcc-8
+      #- g++-8
+
 cache:
   pip: true
   directories:
     - $HOME/.ccache
 
-python:
-  - 2.7
-  - 3.5
-  - 3.2
-  - 3.3
-  - 3.4
-  - 2.6
-  - 3.5-dev
-  - 3.6-dev
-  - pypy
-  - pypy3
-
 env:
   global:
     - USE_CCACHE=1
     - CCACHE_SLOPPINESS=pch_defines,time_macros
     - CCACHE_COMPRESS=1
-    - CCACHE_MAXSIZE=100M
+    - CCACHE_MAXSIZE=250M
     - PATH="/usr/lib/ccache:$PATH"
-  matrix:
-    - BACKEND=c
-    - BACKEND=cpp
-
-branches:
-  only:
-    - master
-
-install:
-  - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build
+    - PYTHON_VERSION=3.8
+    - OS_NAME=ubuntu
 
-before_script: ccache -s
-
-script:
-  - PYTHON_DBG="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg"
-  - if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv Debugger --backends=$BACKEND; fi
-  - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i
-  - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv -x Debugger --backends=$BACKEND -j4
+python: 3.8
 
 matrix:
-  allow_failures:
-    - python: pypy
-    - python: pypy3
-    - python: 3.5-dev
-    - python: 3.6-dev
-  exclude:
-    - python: pypy
+  include:
+    - arch: arm64
+      env: BACKEND=c
+    - arch: arm64
       env: BACKEND=cpp
-    - python: pypy3
+    - arch: ppc64le
+      env: BACKEND=c
+    - arch: ppc64le
       env: BACKEND=cpp
+    # Disabled due to test errors
+    # - arch: s390x
+    #   env: BACKEND=c
+    # - arch: s390x
+    #   env: BACKEND=cpp
 
-addons:
-  apt:
-    packages:
-      - gdb
-      - python-dbg
-      - python3-dbg
+script:
+  - bash ./Tools/ci-run.sh