diff options
| author |   Matti Picus <matti.picus@gmail.com> | 2021-01-27 09:23:07 +0200 |
|---|---|---|
| committer | Matti Picus <matti.picus@gmail.com> | 2021-01-27 09:23:07 +0200 |
| commit | 69ff0d080fb31ceb9526ed0c7a5d56c412a66170 (patch) | |
| tree | 408c62283585bda64166b658f5c73b83f294db2d /pypy | |
| parent | Revert changes to PyFoo_Check and PyFoo_CheckExact signatures (diff) | |
| parent | Fix signature of PyEval_EvalCode (diff) | |
| download | pypy-69ff0d080fb31ceb9526ed0c7a5d56c412a66170.tar.gz pypy-69ff0d080fb31ceb9526ed0c7a5d56c412a66170.tar.bz2 pypy-69ff0d080fb31ceb9526ed0c7a5d56c412a66170.zip | |
merge default into branch
Diffstat (limited to 'pypy')
111 files changed, 2369 insertions, 2810 deletions
diff --git a/pypy/__init__.py b/pypy/__init__.py index ed3bb2a7b2..f2cfdb39c8 100644 --- a/pypy/__init__.py +++ b/pypy/__init__.py @@ -1,17 +1,2 @@ import os pypydir = os.path.realpath(os.path.dirname(__file__)) - -# XXX Should be empty again, soon. -# XXX hack for win64: -# This patch must stay here until the END OF STAGE 1 -# When all tests work, this branch will be merged -# and the branch stage 2 is started, where we remove this patch. -import sys -if hasattr(sys, "maxint") and hasattr(sys, "maxsize"): - if sys.maxint != sys.maxsize: - sys.maxint = sys.maxsize - import warnings - warnings.warn("""\n ----> This win64 port is now in stage 1: sys.maxint was modified. ----> When pypy/__init__.py becomes empty again, we have reached stage 2. -""") diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py index be4934582d..16eae15b45 100644 --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -78,6 +78,15 @@ if sys.platform == "win32": # The _locale module is needed by site.py on Windows default_modules.add("_locale") + # needed to invoke MSVC + translation_modules.update(["thread", "_winreg", "_cffi_backend"]) + + # not ported yet + if IS_64_BITS: + for name in ["cpyext", "_cppyy", "micronumpy"]: + if name in working_modules: + working_modules.remove(name) + if sys.platform == "sunos5": working_modules.remove('fcntl') # LOCK_NB not defined working_modules.remove("_minimal_curses") diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py index 4cade5c6a8..bdb78832b0 100644 --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -75,7 +75,7 @@ master_doc = 'index' # General information about the project. project = u'PyPy' -copyright = u'2020, The PyPy Project' +copyright = u'2021, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -89,7 +89,7 @@ copyright = u'2020, The PyPy Project' # The short X.Y version. version = '7.3' # The full version, including alpha/beta/rc tags. -release = '7.3.3' +release = '7.3.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/pypy/doc/contributing.rst b/pypy/doc/contributing.rst index f5477e5805..959cecb065 100644 --- a/pypy/doc/contributing.rst +++ b/pypy/doc/contributing.rst @@ -40,14 +40,16 @@ details of which can be found in our :ref:`contact <contact>` section. The folks there are very friendly, and can point you in the right direction. We give out commit rights usually fairly liberally, so if you want to do something -with PyPy, you can become a committer. We also run frequent coding sprints which -are separately announced and often happen around Python conferences such as -EuroPython or PyCon. Upcoming events are usually announced on `the blog`_. +with PyPy, you can become a "developer" by logging into https://foss.heptapod.net +and clicking the "Request Access" link on the `PyPy group page`. We also run +coding sprints which are separately announced and are usually announced on `the +blog`_. Further Reading: :ref:`Contact <contact>` .. _the blog: https://morepypy.blogspot.com .. _pypy-dev mailing list: https://mail.python.org/mailman/listinfo/pypy-dev +.. _`PyPy group page`: https://foss.heptapod.net/pypy Your first contribution @@ -96,31 +98,19 @@ Thanks to `Octobus <https://octobus.net/>`_ and `Clever Cloud </a> </h1> -If you are new with Mercurial and Heptapod, you can read this `short tutorial`_ - -.. _`short tutorial`: https://heptapod.net/pages/quick-start-guide.html - -However, we recommend at this time you **not** use topic branches. We prefer -the usual mercurial named branch model, as pointed out in the :ref:`FAQ -<github>` about why we didn't move to git. - Get Access ---------- -The important take-away from that tutorial for experienced developers is that -since the free hosting on foss.heptapod.net does not allow personal forks, you +As stated above, you need to request access to the repo. +Since the free hosting on foss.heptapod.net does not allow personal forks, you need permissions to push your changes directly to our repo. Once you sign in to https://foss.heptapod.net using either a new login or your GitHub or Atlassian logins, you can get developer status for pushing directly to the project (just ask by clicking the link at foss.heptapod.net/pypy just under the logo, and you'll get it, basically). Once you have it you can rewrite your file ``.hg/hgrc`` to contain ``default = ssh://hg@foss.heptapod.net/pypy/pypy``. -Your changes will then be pushed directly to the official repo, but (if you -follow these rules) they are still on a branch, and we can still review the -branches you want to merge. With developer status, you can push topic -branches. If you wish to push long-lived branches, you will need to ask for -higher permissions. - +Your changes will then be pushed directly to a branch on the official repo, and +we will review the branches you want to merge. Clone ----- @@ -138,8 +128,7 @@ Clone then edit ``.hg/hgrc`` as above and do ``hg pull && hg up``. * Now you have a complete copy of the PyPy repo. Make a long-lived branch - with a command like ``hg branch name_of_your_branch``, or make a short- - lived branch for a simple fix with a command like ``hg topic issueXXXX``. + with a command like ``hg branch name_of_your_branch``. Edit ---- diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst index d9e2e1c444..d9250efd26 100644 --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -26,10 +26,10 @@ Contributors Anders Chrigstrom Wim Lavrijsen Eric van Riet Paap + Dan Villiom Podlaski Christiansen Remi Meier Richard Emslie Alexander Schremmer - Dan Villiom Podlaski Christiansen Lukas Diekmann Sven Hager Anders Lehmann @@ -52,6 +52,7 @@ Contributors Lawrence Oluyede Bartosz Skowron Daniel Roberts + Stefano Rivera Adrien Di Mascio Niko Matsakis Alexander Hesse @@ -66,7 +67,6 @@ Contributors Michal Bendowski Jan de Mooij Stefan Beyer - Stefano Rivera Tyler Wade Vincent Legoll Michael Foord @@ -79,348 +79,350 @@ Contributors Devin Jeanpierre Bob Ippolito Bruno Gola + nulano David Malcolm Yannick Jadoul Squeaky - Edd Barrett Timo Paulssen - Laurence Tratt + Edd Barrett Marius Gedminas - Nicolas Truessel + Laurence Tratt Alexandre Fayolle + Nicolas Truessel Simon Burton Martin Matusiak - Wenzhu Man Konstantin Lopuhin + Wenzhu Man John Witulski Jeremy Thurgood - nulano + Dario Bertini Greg Price Ivan Sichmann Freitas - Dario Bertini + Simon Cross Mark Pearse Julian Berman - Simon Cross - Tobias Pape Andreas Stührk + Tobias Pape Jean-Philippe St. Pierre Stian Andreassen Guido van Rossum Pavel Vinogradov William Leslie Paweł Piotr Przeradowski - marky1991 + Paul deGrandis Ilya Osadchiy Tobias Oberstein - Paul deGrandis + marky1991 Boris Feigin - Taavi Burns Adrian Kuhn tav + Taavi Burns Joannah Nanjekye Georg Brandl quejebo Bert Freudenberg + Gerald Klix Wanja Saatkamp Mike Blume - Gerald Klix Oscar Nierstrasz Rami Chowdhury Stefan H. Muller Dodan Mihai + Michał Górny Tim Felgentreff Eugene Oden Colin Valliant - Jeff Terrace Henry Mason - Vasily Kuznetsov - Preston Timmons + Jeff Terrace David Ripton + Preston Timmons + Vasily Kuznetsov Pieter Zieschang - Dusty Phillips Lukas Renggli + Dusty Phillips Guenter Jantzen - Jasper Schulz - Ned Batchelder Amit Regmi + Ned Batchelder + Jasper Schulz Anton Gulenko - Sergey Matyunin - Andrew Chambers - Łukasz Langa + Ben Young Nicolas Chauvat Andrew Durdin - Ben Young + Andrew Chambers + Sergey Matyunin + Łukasz Langa + Nicholas Riley Michael Schneider Yusuke Tsutsumi - Nicholas Riley - Jason Chu - Igor Trindade Oliveira - Yichao Yu - Michael Twomey Rocco Moretti Gintautas Miliauskas + Michael Twomey + Igor Trindade Oliveira + Jason Chu + Yichao Yu Lucian Branescu Mihaila - Mariano Anaya anatoly techtonik - Lin Cheng + Mariano Anaya + Olivier Dormond + Jared Grubb Karl Bartel Gabriel Lavoie - Jared Grubb - Alecsandru Patrascu - Olivier Dormond Wouter van Heyst - Sebastian Pawluś + Alecsandru Patrascu + Lin Cheng Brian Dorsey Victor Stinner Andrews Medina - Aaron Iles - Toby Watson - Daniel Patrick + Sebastian Pawluś Stuart Williams + Toby Watson Antoine Pitrou + Aaron Iles Christian Hudon + Daniel Patrick Justas Sadzevicius + Gasper Zejn Neil Shepperd - Michael Cheng Mikael Schönenberg + Michael Cheng Stanislaw Halik + Berkin Ilbeyi Mihnea Saracin Matt Jackson - Berkin Ilbeyi - Gasper Zejn - Faye Zhao - Elmo Mäntynen + Jonathan David Riehl Anders Qvist + Beatrice During + Elmo Mäntynen Corbin Simpson Chirag Jadwani - Mike Pavone + Faye Zhao Pauli Virtanen - Jonathan David Riehl - Beatrice During - Alex Perry - Robert Zaremba + Mike Pavone Alan McIntyre Alexander Sedov - David C Ellis - Vaibhav Sood - Reuben Cummings - Attila Gobi + Alex Perry Floris Bruynooghe Christopher Pope - Tristan Arthur - Christian Tismer + Attila Gobi + Vaibhav Sood + Reuben Cummings + Robert Zaremba + David C Ellis + Jens-Uwe Mager Dan Stromberg Carl Meyer - Florin Papa - Arianna Avanzini - Jens-Uwe Mager - Matt Billenstein - Valentina Mukhamedzhanova Stefano Parmesan - touilleMan - Anthony Sottile + Alexis Daboville + Christian Tismer Marc Abramowitz Arjun Naik + Valentina Mukhamedzhanova + Florin Papa Aaron Gallagher - Alexis Daboville - Karl Ramm - Lukas Vacek - Omer Katz + touilleMan + Tristan Arthur + Anthony Sottile + Arianna Avanzini + Matt Billenstein Jacek Generowicz - Tomasz Dziopa Sylvain Thenault - Jakub Stasiak - Andrew Dalke Alejandro J. Cura - Vladimir Kryachko - Miro Hrončok + Andrew Dalke Gabriel - Thomas Hisch - Mark Williams - Kunal Grover Nathan Taylor + Karl Ramm + Vladimir Kryachko + Lukas Vacek + Jakub Stasiak + Omer Katz + Kunal Grover + Mark Williams + Thomas Hisch Barry Hart + Tomasz Dziopa + cptpcrd + Lutz Paelike + Ignas Mikalajunas + Martin Blais + Jacob Oscarson + Lene Wagner + Lucio Torre + Henrik Vendelbo + Artur Lisiecki Travis Francis Athougies + Miguel de Val Borro + Kristjan Valur Jonsson + Christoph Gerum Yasir Suhail + Tomo Cocoa + Neil Blakey-Milner + Dan Buch + Lars Wassermann Sergey Kishchenko - Martin Blais - Catalin Fierut - Lutz Paelike + Ryan Gonzalez Ian Foote + David Lievens + Richard Lancaster Philipp Rustemeuer - Bernd Schoeller - olliemath Logan Chien Catalin Gabriel Manciu - Jacob Oscarson - Ryan Gonzalez + Miro Hrončok Antoine Dupre - Kristjan Valur Jonsson - Lucio Torre - Richard Lancaster - Dan Buch - Lene Wagner - Tomo Cocoa - David Lievens - Neil Blakey-Milner - Henrik Vendelbo - Lars Wassermann - Ignas Mikalajunas - Christoph Gerum - Miguel de Val Borro - Artur Lisiecki - joserubiovidales@gmail.com - afteryu - Toni Mattis - Vincent Michel - Laurens Van Houtven - Bobby Impollonia - Roberto De Ioris - Sreepathi Pai - Jeong YunWon - dakarpov@gmail.com - Christopher Armstrong - Bolutife Ogunsola - Aaron Tubbs - Vasantha Ganesh K - Jason Michalski - Radu Ciorba - Ryan Hileman - Markus Holtermann - Georges Racinet + Bernd Schoeller + olliemath + Catalin Fierut + Gustavo Niemeyer Andrew Thompson + Joshua Gilbert Yusei Tahara - Ruochen Huang - Fabio Niephaus + Christopher Armstrong + Anders Sigfridsson + Stephan Busemann + Godefroid Chappelle + Dan Colish Akira Li - Gustavo Niemeyer - joachim-ballmann@bitbucket.org - Nate Bragg - Lucas Stadler + Bobby Impollonia roberto@goyle - Carl Bordum Hansen - Matt Bogosian + Roberto De Ioris + timo + Anna Katrina Dominguez + Juan Francisco Cantero Hurtado + Ben Darnell + Rafał Gałczyński Yury V. Zaytsev - florinpapa - Anders Sigfridsson - Nikolay Zinov + Laurens Van Houtven rafalgalczynski@gmail.com - Joshua Gilbert - Anna Katrina Dominguez + Jason Michalski + Toni Mattis + Lucas Stadler + Jeong YunWon + Ruochen Huang + Markus Holtermann Kim Jin Su + Matt Bogosian + Aaron Tubbs Amber Brown + Nikolay Zinov + florinpapa + Vasantha Ganesh K + Fabio Niephaus + Nate Bragg + afteryu Andrew Stepanov - ashwinahuja - Rafał Gałczyński - Ben Darnell - Juan Francisco Cantero Hurtado - Godefroid Chappelle + Radu Ciorba + Carl Bordum Hansen Paul Ganssle Michal Kuffa - Stephan Busemann - Bystroushaak - Dan Colish + joachim-ballmann@bitbucket.org + Vincent Michel Ram Rachum - timo - Volodymyr Vladymyrov - Daniel Neuhäuser - Flavio Percoco - halgari - Michał Górny + Bystroushaak + Ryan Hileman + joserubiovidales@gmail.com + dakarpov@gmail.com + Sreepathi Pai + Georges Racinet + Bolutife Ogunsola + ashwinahuja + cjmcdonald@google.com + Anna Ravencroft + Dinu Gherman + Michael Chermside Jim Baker + Zooko Wilcox-O Hearn + Daniel Neuhäuser + Konrad Delong + Rodrigo Araújo + Armin Ronacher + Jim Hunziker + Christian Muirhead + Brett Cannon Chris Lambacher - John Aldis + Dan Loewenherz coolbutuseless@gmail.com - Yasen Kiprov + Christopher Groskopf + Buck Golemon + soareschen + Even Wiik Thomassen + Antony Lee + James Lan + yrttyr + Kristoffer Kleine + Julien Phalip + shoma hosaka + Tomer Chachamu + Flavio Percoco + Markus Unterwaditzer Mike Bayer - mark doerr - Rodrigo Araújo - Daniil Yarancev - Min RK OlivierBlanvillain - bernd.schoeller@inf.ethz.ch - Jonas Pfannschmidt - Alex Kashirin - Zearin - Joannah Nanjekye nanjekyejoannah@gmail.com - Ihar Shabes - Johan Forsberg - Andrey Churin + jiaaro + James Robert + aliceinwire + Kurt Griffiths + Matthew Miller + Asmo Soinio + Stefan Marr + Boglarka Vezer + Mads Kiilerich Dan Crosta - reubano@gmail.com - Stanisław Halik - DeVerne Jones - Julien Phalip + Dan Sanders + Ben Mather + Chris Pressey + halgari + Berker Peksag Roman Podoliaka - Steve Papanik - Henri Tuhola + Nikolaos-Digenis Karagiannis + Donald Stufft + Volodymyr Vladymyrov + Andrey Churin + Niclas Olofsson + Yaroslav Fedevych + Zearin + Tobias Diaz + Jason Madden + Jonas Pfannschmidt + werat + JohnDoe + Diana Popa Eli Stevens - Boglarka Vezer - gabrielg@ec2-54-146-239-158.compute-1.amazonaws.com + pizi + remarkablerocket + reubano@gmail.com + Daniil Yarancev PavloKapyshin - shaolo1 - Hervé Beraud - Tomer Chachamu - Christopher Groskopf - Asmo Soinio - Antony Lee - Jim Hunziker - shoma hosaka - Buck Golemon - whitequark + Graham Markall + Stanisław Halik Iraklis D. - JohnDoe - yrttyr - Michael Chermside - Anna Ravencroft - remarkablerocket - Ivan Petre Vijiac - Berker Peksag - Christian Muirhead - soareschen - Matthew Miller - Jesdi - Konrad Delong - Dinu Gherman - Sam Edwards - pizi - Tomáš Pružina - James Robert - Chris AtLee - Armin Ronacher - Diana Popa - Mike Kaplinskiy - Mads Kiilerich - Brett Cannon + Min RK Caleb Hattingh - aliceinwire - Zooko Wilcox-O Hearn - James Lan - jiaaro - Evgenii Gorinov - Markus Unterwaditzer - Kristoffer Kleine - Graham Markall - paugier - Dan Loewenherz - werat - Tomas Hrnciar + Steve Papanik + m@funkyhat.org + Tomáš Pružina + gabrielg@ec2-54-146-239-158.compute-1.amazonaws.com Filip Salomonsson - Niclas Olofsson - Zsolt Cserna - Chris Pressey - Tobias Diaz + Johan Forsberg + Evgenii Gorinov + John Aldis + Hervé Beraud Paul Graydon + whitequark + DeVerne Jones + Zsolt Cserna + Yasen Kiprov mkuffa - Nikolaos-Digenis Karagiannis - Kurt Griffiths - Ben Mather - Donald Stufft + Ivan + Jesdi + paugier + bernd.schoeller@inf.ethz.ch + Sam Edwards + Ihar Shabes kotus9 - Dan Sanders - Jason Madden - Yaroslav Fedevych - Even Wiik Thomassen - m@funkyhat.org - Stefan Marr + mark doerr + Tomas Hrnciar + Joannah Nanjekye nanjekyejoannah@gmail.com + Alex Kashirin + Mike Kaplinskiy + Henri Tuhola + shaolo1 + Chris AtLee diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst index 4d2dad06be..66ed78980d 100644 --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -365,14 +365,17 @@ Miscellaneous implementation detail that shows up because of internal C-level slots that PyPy does not have. -* on CPython, ``[].__add__`` is a ``method-wrapper``, and - ``list.__add__`` is a ``slot wrapper``. On PyPy these are normal - bound or unbound method objects. This can occasionally confuse some +* on CPython, ``[].__add__`` is a ``method-wrapper``, ``list.__add__`` + is a ``slot wrapper`` and ``list.extend`` is a (built-in) ``method`` + object. On PyPy these are all normal method or function objects (or + unbound method objects on PyPy2). This can occasionally confuse some tools that inspect built-in types. For example, the standard library ``inspect`` module has a function ``ismethod()`` that returns True on unbound method objects but False on method-wrappers or slot - wrappers. On PyPy we can't tell the difference, so - ``ismethod([].__add__) == ismethod(list.__add__) == True``. + wrappers. On PyPy we can't tell the difference. So on PyPy2 we + have ``ismethod([].__add__) == ismethod(list.extend) == True``; + on PyPy3 we have ``isfunction(list.extend) == True``. On CPython + all of these are False. * in CPython, the built-in types have attributes that can be implemented in various ways. Depending on the way, if you try to @@ -422,7 +425,8 @@ Miscellaneous probably be ignored by an implementation of ``sys.getsizeof()``, but their overhead is important in some cases if they are many instances with unique maps. Conversely, equal strings may share their internal - string data even if they are different objects---or empty containers + string data even if they are different objects---even a unicode string + and its utf8-encoded ``bytes`` version are shared---or empty containers may share parts of their internals as long as they are empty. Even stranger, some lists create objects as you read them; if you try to estimate the size in memory of ``range(10**6)`` as the sum of all diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst index df406bdda4..64d098b09d 100644 --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -379,7 +379,7 @@ interpreter; preliminary versions of a `JavaScript interpreter`_ (produced during a sprint). On the `PyPy bitbucket page`_ there is also a Scheme and an Io implementation; both of these are unfinished at the moment. -.. _Topaz: https://docs.topazruby.com/en/latest/ +.. _Topaz: https://github.com/topazproject/topaz .. _Hippy: https://morepypy.blogspot.ch/2012/07/hello-everyone.html .. _JavaScript interpreter: https://bitbucket.org/pypy/lang-js/ .. _Prolog interpreter: https://bitbucket.org/cfbolz/pyrolog/ diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst index 43cc823fcb..b7ff8de72c 100644 --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -96,6 +96,10 @@ Other steps create a fresh whatsnew_head.rst after the release and add the new file to pypy/doc/index-of-whatsnew.rst + * rename pypy/doc/whatsnew-pypy3-HEAD.rst to whatsnew-pypy3-VERSION.rst + create a fresh whatsnew-pypy3_HEAD.rst after the release + and add the new file to pypy/doc/index-of-whatsnew.rst + * write release announcement pypy/doc/release-VERSION.rst The release announcement should contain a direct link to the download page @@ -121,11 +125,16 @@ Other steps (it is important to mark this as a candidate since usually at least two tries are needed to complete the process) and download and repackage source from the buildbot. You may find it convenient to use the ``repackage.sh`` - script in pypy/tool/release to do this. + script in ``pypy/tool/release`` to do this. Also repackage and upload source "-src.tar.bz2" - * Upload binaries to https://buildbot.pypy.org/mirror + * Upload binaries to https://buildbot.pypy.org/mirror. Add the files to + the ``versions.json`` in ``pypy/tools/release``, upload it, and run the + ``check_versions.py`` file in that directory. This file is used by various + downstream tools like "github actions" to find valid pypy downloads. It + takes an hour for + https://downloads.python.org/pypy/ to sync * Send out a mailing list message asking for last-minute comments and testing @@ -138,9 +147,12 @@ Other steps * send announcements to twitter.com, pypy-dev, python-list, python-announce, python-dev ... -* If all is OK, document the released version +* If all is OK, document the released version and suggest popular tools update + to support it. Github actions will pick up the versions.json. * add a tag on the codespeed web site that corresponds to pypy release * revise versioning at https://readthedocs.org/projects/pypy - * tag the final release(s) with appropriate tags + * suggest updates to multibuild_ and cibuildwheel_ +.. _multibuild: https://github.com/matthew-brett/multibuild +.. _cibuildwheel: https://github.com/joerick/cibuildwheel diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst index d14307a260..78af582e5b 100644 --- a/pypy/doc/index-of-release-notes.rst +++ b/pypy/doc/index-of-release-notes.rst @@ -6,6 +6,7 @@ Combined releases .. toctree:: + release-v7.3.3.rst release-v7.3.2.rst release-v7.3.1.rst release-v7.3.0.rst diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst index 64061678a0..1770829680 100644 --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ CPython 2.7 compatible versions .. toctree:: whatsnew-head.rst + whatsnew-pypy2-7.3.3.rst whatsnew-pypy2-7.3.2.rst whatsnew-pypy2-7.3.1.rst whatsnew-pypy2-7.3.0.rst @@ -46,6 +47,7 @@ CPython 3.6 compatible versions .. toctree:: whatsnew-pypy3-head.rst + whatsnew-pypy3-7.3.3.rst whatsnew-pypy3-7.3.2.rst whatsnew-pypy3-7.3.1.rst whatsnew-pypy3-7.3.0.rst diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst index 9eeabedd49..431d53afae 100644 --- a/pypy/doc/project-ideas.rst +++ b/pypy/doc/project-ideas.rst @@ -150,26 +150,19 @@ knowledge of the internals. Head over to `vmprof-python`_, `vmprof-server`_ and .. _vmprof-server: https://github.com/vmprof/vmprof-server .. _vmprof-integration: https://github.com/vmprof/vmprof-integration -Optimized Unicode Representation --------------------------------- - -CPython 3.3 will use an optimized unicode representation (see :pep:`0393`) which switches between -different ways to represent a unicode string, depending on whether the string -fits into ASCII, has only two-byte characters or needs four-byte characters. - -The actual details would be rather different in PyPy, but we would like to have -the same optimization implemented. - -Or maybe not. We can also play around with the idea of using a single -representation: as a byte string in utf-8. (This idea needs some extra logic -for efficient indexing, like a cache.) Work has begun on the ``unicode-utf`` -and ``unicode-utf8-py3`` branches. More is needed, for instance there are -SIMD optimizations that are not yet used. - Convert RPython to Python3 -------------------------- -The world is moving on, we should too. +The world is moving on, we should too. Work in this direction has begun on the +``rpython3`` branch, mainly to enable building documentation with Python3. Some +things that are known to need careful refactoring: + +- a single character in python3 is an int, not a byte +- we use ``str``/``unicode`` to distiguish between different modes of + operation for windows in ``make_win32_traits``. + +There are probably more. The branch currently does not pass rpython tests so +work is needed to back out some of the changes and redo them properly Improve performance ------------------- @@ -273,7 +266,10 @@ and it is hard to imagine NumPy abandoning the C-API. Here are a few ideas: Support more platforms ---------------------- -We have a plan for a `Windows 64`_ port. +We have a plan for a `Windows 64`_ port. There is progress on the ``win64`` +branch. Help is needed to continue the work. Stage I is complete: we now have +a 64-bit PyPy2.7 on windows. But it is missing cpyext and other tidbits to +enable releasing it. .. _`Windows 64`: windows.html#what-is-missing-for-a-full-64-bit-translation @@ -322,3 +318,6 @@ good work that needs to be finished: TODO: see the end of the blog post +Work has begun on HPy_ to enable a faster C-API. + +.. _HPy: https://hpy.readthedocs.io/en/latest/ diff --git a/pypy/doc/release-v7.3.3.rst b/pypy/doc/release-v7.3.3.rst new file mode 100644 index 0000000000..7671402ed8 --- /dev/null +++ b/pypy/doc/release-v7.3.3.rst @@ -0,0 +1,195 @@ +============================================== +PyPy v7.3.3: release of 2.7, 3.6, and 3.7 beta +============================================== + +-The PyPy team is proud to release the version 7.3.3 of PyPy, which includes +three different interpreters: + + - PyPy2.7, which is an interpreter supporting the syntax and the features of + Python 2.7 including the stdlib for CPython 2.7.18 (updated from the + previous version) + + - PyPy3.6: which is an interpreter supporting the syntax and the features of + Python 3.6, including the stdlib for CPython 3.6.12 (updated from the + previous version). + + - PyPy3.7 beta: which is our second release of an interpreter supporting the + syntax and the features of Python 3.7, including the stdlib for CPython + 3.7.9. We call this beta quality software, there may be issues about + compatibility with new and changed features in CPython 3.7. + Please let us know what is broken or missing. We have not implemented the + `documented changes`_ in the ``re`` module, and a few other pieces are also + missing. For more information, see the `PyPy 3.7 wiki`_ page + +The interpreters are based on much the same codebase, thus the multiple +release. This is a micro release, all APIs are compatible with the 7.3 +releases, but read on to find out what is new. + +.. + The major new feature is prelminary support for the Universal mode of HPy: a + new way of writing c-extension modules to totally encapsulate the `PyObject*`. + The goal, as laid out in the `HPy blog post`_, is to enable a migration path + for c-extension authors who wish their code to be performant on alternative + interpreters like GraalPython_ (written on top of the Java virtual machine), + RustPython_, and PyPy. Thanks to Oracle for sponsoring work on HPy. + +Several issues exposed in the 7.3.2 release were fixed. Many of them came from the +great work ongoing to ship PyPy-compatible binary packages in `conda-forge`_. +A big shout out to them for taking this on. + +Development of PyPy has moved to https://foss.heptapod.net/pypy/pypy. +This was covered more extensively in this `blog post`_. We have seen an +increase in the number of drive-by contributors who are able to use gitlab + +mercurial to create merge requests. + +The `CFFI`_ backend has been updated to version 1.14.3. We recommend using CFFI +rather than c-extensions to interact with C, and using cppyy_ for performant +wrapping of C++ code for Python. + +A new contributor took us up on the challenge to get `windows 64-bit`_ support. +The work is proceeding on the ``win64`` branch, more help in coding or +sponsorship is welcome. In anticipation of merging this large change, we fixed +many test failures on windows. + +As always, this release fixed several issues and bugs. We strongly recommend +updating. Many of the fixes are the direct result of end-user bug reports, so +please continue reporting issues as they crop up. + +You can find links to download the v7.3.3 releases here: + + https://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project. If PyPy is not quite good enough for your needs, we are available for +direct consulting work. + +We would also like to thank our contributors and encourage new people to join +the project. PyPy has many layers and we need help with all of them: `PyPy`_ +and `RPython`_ documentation improvements, tweaking popular modules to run +on pypy, or general `help`_ with making RPython's JIT even better. Since the +previous release, we have accepted contributions from 2 new contributors, +thanks for pitching in. + +If you are a python library maintainer and use c-extensions, please consider +making a cffi / cppyy version of your library that would be performant on PyPy. +In any case both `cibuildwheel`_ and the `multibuild system`_ support +building wheels for PyPy. + +.. _`PyPy`: index.html +.. _`RPython`: https://rpython.readthedocs.org +.. _`help`: project-ideas.html +.. _`CFFI`: https://cffi.readthedocs.io +.. _`cppyy`: https://cppyy.readthedocs.io +.. _`multibuild system`: https://github.com/matthew-brett/multibuild +.. _`cibuildwheel`: https://github.com/joerick/cibuildwheel +.. _`blog post`: https://morepypy.blogspot.com/2020/02/pypy-and-cffi-have-moved-to-heptapod.html +.. _`conda-forge`: https://conda-forge.org/blog//2020/03/10/pypy +.. _`documented changes`: https://docs.python.org/3/whatsnew/3.7.html#re +.. _`PyPy 3.7 wiki`: https://foss.heptapod.net/pypy/pypy/-/wikis/py3.7%20status +.. _`wheels on PyPI`: https://pypi.org/project/numpy/#files +.. _`windows 64-bit`: https://foss.heptapod.net/pypy/pypy/-/issues/2073#note_141389 +.. _`HPy blog post`: https://morepypy.blogspot.com/2019/12/hpy-kick-off-sprint-report.html +.. _`GraalPython`: https://github.com/graalvm/graalpython +.. _`RustPython`: https://github.com/RustPython/RustPython + + +What is PyPy? +============= + +PyPy is a Python interpreter, a drop-in replacement for CPython 2.7, 3.6, and +3.7. It's fast (`PyPy and CPython 3.7.4`_ performance +comparison) due to its integrated tracing JIT compiler. + +We also welcome developers of other `dynamic languages`_ to see what RPython +can do for them. + +This PyPy release supports: + + * **x86** machines on most common operating systems + (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD) + + * big- and little-endian variants of **PPC64** running Linux, + + * **s390x** running Linux + + * 64-bit **ARM** machines running Linux. + +PyPy does support ARM 32 bit processors, but does not release binaries. + +.. _`PyPy and CPython 3.7.4`: https://speed.pypy.org +.. _`dynamic languages`: https://rpython.readthedocs.io/en/latest/examples.html + +Changelog +========= + +Changes shared across versions +------------------------------ +- Better support macOSx <= 10.13 +- Use ``MACOSX_DEPLOYMENT_TARGET`` from environment, if set, when translating +- Update stdlib to 2.7.18 +- Add limited support for ``long double`` to RPython (issue 3312_) +- Package a ``.hg_archival.txt`` in the source tarball. The file was left out + of the source tarball after the move to heptapod (issue 3315_) +- Fix a race condition in ``crypt`` +- Update expired certificates used in python testing +- Always use NT ``sysconfig`` scheme on windows (issue 3321_) +- Delay importing ``os`` and ``_codecs`` until after importing ``site`` +- `bpo-35194`_: Fix a wrong constant in cp932 codec +- `bpo-34794`_: Fix a leak in Tkinter +- `bpo-33781`_: ``audioop``: enhance rounding double as int +- `bpo-31893`_: Simplify ``select.kqueue`` object comparison + +C-API (cpyext) and c-extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Check for ``None`` in ``PyLong_AsUnsignedLongLong`` +- Dynamically allocate ``Py_buffer.format`` if needed (issue 3336_) +- Fix for readonly flag on PyObject_GetBuffer(<bytes>, view) (issue 3307_) + +Python 3.6+ +----------- +- Fix windows support for using ``venv`` inside a ``venv`` +- Support ``os.sendfile`` on BSD derivatives (macOSx and freebsd) +- Fix ``_socket.sethostname(<bytes>)`` +- Fix for modifying ``self.__dict__`` in ``self.__set_name__``, issue 3326_ +- bpo-33041_: Fixed jumping if the function contains an ``async for`` loop. +- bpo-17288_: Prevent jump from a yield statement +- bpo-11471_: avoid generating a ``JUMP_FORWARD`` instruction at the end of an + ``if``-block if there is no ``else``-clause +- Fix ``os.listdir('')`` and ``os.stat('')`` on windows (issue 3331_) +- Fix many unicode encoding/decoding errors on windows +- Fix pickling of time subclasses (issue 3324_, bpo-41966_) +- Add support for ``sqlite3_load_extension`` (issue 3334_) +- Change default file encoding from mbcs to utf-8 on windows +- Change default file encoding from ascii to utf-8 on linux +- Add ``resource.prlimit()`` +- Accept PathLike in ``nt._getfullpathname`` (issue 3343_) +- Fix some problems with ``winreg`` + + +Python 3.6 C-API +~~~~~~~~~~~~~~~~ + +- Export ``PyStructSequence_NewType`` (issue 3346_) + +.. _3312: https://foss.heptapod.net/pypy/pypy/-/issues/3312 +.. _3315: https://foss.heptapod.net/pypy/pypy/-/issues/3315 +.. _3321: https://foss.heptapod.net/pypy/pypy/-/issues/3321 +.. _3326: https://foss.heptapod.net/pypy/pypy/-/issues/3326 +.. _3331: https://foss.heptapod.net/pypy/pypy/-/issues/3331 +.. _3324: https://foss.heptapod.net/pypy/pypy/-/issues/3324 +.. _3334: https://foss.heptapod.net/pypy/pypy/-/issues/3334 +.. _3336: https://foss.heptapod.net/pypy/pypy/-/issues/3336 +.. _3307: https://foss.heptapod.net/pypy/pypy/-/issues/3307 +.. _3343: https://foss.heptapod.net/pypy/pypy/-/issues/3343 +.. _3346: https://foss.heptapod.net/pypy/pypy/-/issues/3346 + +.. _`merge request 723`: https://foss.heptapod.net/pypy/pypy/-/merge_request/723 + +.. _bpo-35194: https://bugs.python.org/issue35194 +.. _bpo-34794: https://bugs.python.org/issue34794 +.. _bpo-33781: https://bugs.python.org/issue33781 +.. _bpo-31893: https://bugs.python.org/issue31893 +.. _bpo-33041: https://bugs.python.org/issue33041 +.. _bpo-17288: https://bugs.python.org/issue17288 +.. _bpo-11471: https://bugs.python.org/issue11471 +.. _bpo-41966: https://bugs.python.org/issue41966 diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst index d48a0c6edf..01545d5ffc 100644 --- a/pypy/doc/sandbox.rst +++ b/pypy/doc/sandbox.rst @@ -4,10 +4,13 @@ PyPy's sandboxing features ========================== .. warning:: This describes the old, unmaintained version. A new version - is in progress and should be merged back to trunk at some point soon. + is in progress in the ``sandbox-2`` and ``py3.6-sandbox-2`` branches and in + the sandboxlib_ repo. Please see its description here: https://mail.python.org/pipermail/pypy-dev/2019-August/015797.html + Also note that python 3.7+ requires the _thread module, which may be + a consideration in escaping the sandbox. Introduction ------------ @@ -30,6 +33,7 @@ attacker is allowed to run (why? read about pysandbox_). .. _SECCOMP: https://code.google.com/p/seccompsandbox/wiki/overview .. _pysandbox: https://mail.python.org/pipermail/python-dev/2013-November/130132.html +.. _sandbloxlib: https://foss.heptapod.net/pypy/sandboxlib Another point of comparison: if we were instead to try to plug CPython into a special virtualizing C library, we would get a result diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py index 9f8bef4e5d..f617c199e1 100644 --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -1,3 +1,4 @@ +from __future__ import print_function # NOTE: run this script with LANG=en_US.UTF-8 # works with pip install mercurial==3.0 @@ -94,7 +95,7 @@ alias = { } alias_map = {} -for name, nicks in alias.iteritems(): +for name, nicks in alias.items(): for nick in nicks: alias_map[nick] = name @@ -132,13 +133,13 @@ def get_more_authors(log): def main(show_numbers): ui = mercurial.ui.ui() - repo = mercurial.hg.repository(ui, str(ROOT)) + repo = mercurial.hg.repository(ui, str(ROOT).encode('utf-8')) authors_count = defaultdict(int) for i in repo: ctx = repo[i] authors = set() - authors.add(get_canonical_author(ctx.user())) - authors.update(get_more_authors(ctx.description())) + authors.add(get_canonical_author(ctx.user().decode('utf-8'))) + authors.update(get_more_authors(ctx.description().decode('utf-8'))) for author in authors: if author not in excluded: authors_count[author] += 1 @@ -153,13 +154,13 @@ def main(show_numbers): ## else: ## print name - items = authors_count.items() + items = list(authors_count.items()) items.sort(key=operator.itemgetter(1), reverse=True) for name, n in items: if show_numbers: - print '%5d %s' % (n, name) + print('%5d %s' % (n, name)) else: - print ' ' + name + print(' ' + name) if __name__ == '__main__': show_numbers = '-n' in sys.argv diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst index bbf2648f71..f8adb5e3c2 100644 --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -1,23 +1,51 @@ ============================ -What's new in PyPy2.7 7.3.2+ +What's new in PyPy2.7 7.3.3+ ============================ -.. this is a revision shortly after release-pypy-7.3.2 -.. startrev: c136fdb316e4 +.. this is a revision shortly after release-pypy-7.3.3 +.. startrev: de512cf13506 +.. branch: new-ci-image -.. branch: cross_compilation_fixes +CI: Add a Dockerfile for CI to prevent hitting pull limits on docker hub -Respect PKG_CONFIG and CC in more places to allow cross-compilation +.. branch: issue-3333 -.. branch: darwin-sendfile-2.7 +Fix xml.etree.ElementTree assigning default attribute values: issue 3333 -Add posix.sendfile to darwin for python3.6+ +.. branch: rpython-rsre-for-37 -.. branch: app_main +Support for the new format of regular expressions in Python 3.7 -Avoid using ``import os`` until after ``import site`` in ``app_main`` +.. branch: rpy-cparser -.. branch: stdlib-2.7.18-3 +Upstream internal cparser tool from pypy/ to rpython/ -Update lib-python/2.7 to stdlib-2.7.18 and fix many tests + +.. branch: win64 + +Change rpython and pypy to enable translating 64-bit windows + + +.. branch: rpython-error_value + +Introduce @rlib.objectmodel.llhelper_error_value, will be used by HPy + +.. branch: add-rffi-constcharpsize2str + +Add ``rffi.constcharpsize2str`` + +.. branch: document-win64 + +Refactor documentation of win64 from future plans to what was executed + +.. branch: sync-distutils + +Backport msvc detection from python3, which probably breaks using Visual Studio +2008 (MSVC9, or the version that used to be used to build CPython2.7 on +Windows) + +.. branch: py2.7-winreg + +Backport fixes to winreg adding reflection and fix for passing None (bpo +21151). diff --git a/pypy/doc/whatsnew-pypy2-7.3.3.rst b/pypy/doc/whatsnew-pypy2-7.3.3.rst new file mode 100644 index 0000000000..f3cd29dcd1 --- /dev/null +++ b/pypy/doc/whatsnew-pypy2-7.3.3.rst @@ -0,0 +1,27 @@ +=========================== +What's new in PyPy2.7 7.3.3 +=========================== + +.. this is a revision shortly after release-pypy-7.3.2 +.. startrev: c136fdb316e4 + + +.. branch: cross_compilation_fixes + +Respect PKG_CONFIG and CC in more places to allow cross-compilation + +.. branch: darwin-sendfile-2.7 + +Add posix.sendfile to darwin for python3.6+ + +.. branch: app_main + +Avoid using ``import os`` until after ``import site`` in ``app_main`` + +.. branch: stdlib-2.7.18-3 + +Update lib-python/2.7 to stdlib-2.7.18 and fix many tests + +.. branch: cptpcrd-resource-prlimit + +Add ``prlimit`` to ``resource`` diff --git a/pypy/doc/whatsnew-pypy3-7.3.2.rst b/pypy/doc/whatsnew-pypy3-7.3.2.rst index 34ae286f0f..1aa9649b32 100644 --- a/pypy/doc/whatsnew-pypy3-7.3.2.rst +++ b/pypy/doc/whatsnew-pypy3-7.3.2.rst @@ -46,3 +46,8 @@ of W_IntObjects caused the list to loose its int optimization (issue #3250). Add ``os.sched_rr_get_interval``, ``os.sched_getscheduler``, ``sched_setscheduler``, ``sched_getparam`` + +.. branch: hpy + +Provide a backend for HPy, currently disabled in the release + diff --git a/pypy/doc/whatsnew-pypy3-7.3.3.rst b/pypy/doc/whatsnew-pypy3-7.3.3.rst new file mode 100644 index 0000000000..a1e7fc674a --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-7.3.3.rst @@ -0,0 +1,20 @@ +========================= +What's new in PyPy3 7.3.3 +========================= + +.. branch: stdlib-3.6.12 + +Update the stdlib to 3.6.12 + +.. branch: cptpcrd-resource-prlimit + +Add ``os.prlimit`` if supported by glibc + +.. branch: cptpcrd-resource-py3.6-exceptions + +Rework exception handling in 'resource' module to match CPython + +.. branch: hpy + +Provide a backend for HPy + diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst index ea41c9ad25..8242f92fab 100644 --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -1,11 +1,8 @@ ========================== -What's new in PyPy3 7.3.2+ +What's new in PyPy3 7.3.3+ ========================== -.. this is the revision after release-pypy3.6-v7.3.2 -.. startrev: 9e32f74fc751 +.. this is the revision after release-pypy3.6-v7.3.3 +.. startrev: a57ea1224248 -.. branch: hpy - -Provide a backend for HPy diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst index a307516c46..441841beb1 100644 --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -5,12 +5,8 @@ RPython is supported on Windows platforms, starting with Windows 2000. The following text gives some hints about how to translate a interpreter written in RPython, using PyPy as an example. -PyPy supports only being translated as a 32bit program, even on -64bit Windows. See at the end of this page for what is missing -for a full 64bit translation. - To build pypy-c you need a working python environment, and a C compiler. -It is possible to translate with a CPython 2.6 or later, but this is not +It is possible to translate with a CPython 2.6 or 2.7, but this is not the preferred way, because it will take a lot longer to run – depending on your architecture, between two and three times as long. So head to `our downloads`_ and get the latest stable version. @@ -41,13 +37,15 @@ PyPy has a chicken and egg problem: in order to compile something we need ``msvc.py`` in ``rpython/tools/setuptools_msvc.py``. PyPy will prefer to compile with the latest MSVC compiler it can find, which is -a departure from CPython's desire to compile with Visual Studio 9. +a departure from CPython's desire to compile with the compiler used to compile +the exe in use. Translating PyPy with Visual Studio ----------------------------------- We routinely test translation of PyPy using Visual Studio 2019, MSVC160. -Other configurations may work as well. +Other configurations may work as well. You must use at least Visual Studio +2012. The translation scripts will set up the appropriate environment variables for the compiler, so you do not need to run vcvars before translation. @@ -64,8 +62,7 @@ If you wish to override this detection method to use a different compiler * set the `CC` environment variable to compiler exe to be used, for a different version of MSVC `SET CC=cl.exe`. -**Note:** The RPython translator does currently not support 64 bit Python, and -translation will fail in this case. +**Note:** The RPython translator requires a special 64 bit Python, see below Python and a C compiler are all you need to build pypy, but it will miss some modules that relies on third-party libraries. See below how to get @@ -159,7 +156,7 @@ If you wish to experiment with win64, you must run configure with flags:: or such, depending on your mingw64 download. -hacking on PyPy with the mingw compiler +Hacking on PyPy with the mingw compiler ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since hacking on PyPy means running tests, you will need a way to specify the mingw compiler when hacking (as opposed to translating). As of @@ -175,76 +172,6 @@ environment variable CC to the compiler exe, testing will use it. What is missing for a full 64-bit translation --------------------------------------------- -The main blocker is that we assume that the integer type of RPython is -large enough to (occasionally) contain a pointer value cast to an -integer. The simplest fix is to make sure that it is so, but it will -give the following incompatibility between CPython and PyPy on Win64: - -CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1`` - -PyPy: ``sys.maxint == sys.maxsize == 2**63-1`` - -...and, correspondingly, PyPy supports ints up to the larger value of -sys.maxint before they are converted to ``long``. The first decision -that someone needs to make is if this incompatibility is reasonable. - -Assuming that it is, the first thing to do is probably to hack *CPython* -until it fits this model: replace the field in PyIntObject with a ``long -long`` field, and change the value of ``sys.maxint``. This might just -work, even if half-brokenly: I'm sure you can crash it because of the -precision loss that undoubtedly occurs everywhere, but try not to. :-) - -Such a hacked CPython is what you'll use in the next steps. We'll call -it CPython64/64. - -It is probably not too much work if the goal is only to get a translated -PyPy executable, and to run all tests before translation. But you need -to start somewhere, and you should start with some tests in -``rpython/translator/c/test/``, like ``test_standalone.py`` and -``test_newgc.py``: try to have them pass on top of CPython64/64. - -Keep in mind that this runs small translations, and some details may go -wrong. The most obvious one is to check that it produces C files that -use the integer type ``Signed`` --- but what is ``Signed`` defined to? -It should be equal to ``long`` on every other platform, but on Win64 it -should be something like ``long long``. - -What is more generally needed is to review all the C files in -``rpython/translator/c/src`` for the word ``long``, because this means a -32-bit integer even on Win64. Replace it with ``Signed`` most of the -times. You can replace one with the other without breaking anything on -any other platform, so feel free to. - -Then, these two C types have corresponding RPython types: ``rffi.LONG`` -and ``lltype.Signed`` respectively. The first should really correspond -to the C ``long``. Add tests that check that integers cast to one -type or the other really have 32 and 64 bits respectively, on Win64. - -Once these basic tests work, you need to review ``rpython/rlib/`` for -uses of ``rffi.LONG`` versus ``lltype.Signed``. The goal would be to -fix some more ``LONG-versus-Signed`` issues, by fixing the tests --- as -always run on top of CPython64/64. Note that there was some early work -done in ``rpython/rlib/rarithmetic`` with the goal of running all the -tests on Win64 on the regular CPython, but I think by now that it's a -bad idea. Look only at CPython64/64. - -The major intermediate goal is to get a translation of PyPy with ``-O2`` -with a minimal set of modules, starting with ``--no-allworkingmodules``; -you need to use CPython64/64 to run this translation too. Check -carefully the warnings of the C compiler at the end. By default, MSVC -reports a lot of mismatches of integer sizes as warnings instead of -errors. - -Then you need to review ``pypy/module/*/`` for ``LONG-versus-Signed`` -issues. At some time during this review, we get a working translated -PyPy on Windows 64 that includes all ``--translationmodules``, i.e. -everything needed to run translations. Once we have that, the hacked -CPython64/64 becomes much less important, because we can run future -translations on top of this translated PyPy. As soon as we get there, -please *distribute* the translated PyPy. It's an essential component -for anyone else that wants to work on Win64! We end up with a strange -kind of dependency --- we need a translated PyPy in order to translate a -PyPy ---, but I believe it's ok here, as Windows executables are -supposed to never be broken by newer versions of Windows. - -Happy hacking :-) +This is a placeholder for old links to this topic. We have :ref:`solved the +64-bit translation problems <windows64>` and there are nightly builds of 64-bit windows. + diff --git a/pypy/doc/windows64.rst b/pypy/doc/windows64.rst new file mode 100644 index 0000000000..715676cb4e --- /dev/null +++ b/pypy/doc/windows64.rst @@ -0,0 +1,85 @@ +:orphan: + +.. _windows64: + +64 bit PyPy and Windows +======================= + +Getting 64-bit windows to work with PyPy was a long-standing request that was +finally cracked in the otherwise cursed year of 2020. The problem is that we +assume that the integer type of RPython (``rffi.Signed``) is +large enough to (occasionally) contain a pointer value cast to an +integer. On most platforms the corresponding C type of ``long`` satisfies this +condition. But on 64-bit windows, a ``long`` is 32-bits, while +``sizeof(void*)`` is 64-bits. The simplest fix is to make sure that +``rffi.Signed`` can hold a 64-bit integer, which resutls in a python2 with the +following incompatibility between CPython and PyPy on Win64: + +CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1`` + +PyPy: ``sys.maxint == sys.maxsize == 2**63-1`` + +...and, correspondingly, PyPy2 supports ints up to the larger value of +sys.maxint before they are converted to ``long``. + +What we did +----------- + +The first thing done was to do hack a *CPython2* +until it fits this model: replace the field in PyIntObject with a ``long +long`` field, and change the value of ``sys.maxint``. This is available in +`nulano's branch of cpython`_ (again: this is **python2**). + +This hacked pyton was used in the next steps. We'll call it CPython64/64. + +First the tests in +``rpython/translator/c/test/``, like ``test_standalone.py`` and +``test_newgc.py`` were made to pass on top of CPython64/64. + +This runs small translations, and some details were +wrong. The most obvious one is to make +the integer type ``Signed`` +equal to ``long`` on every other platform, but on Win64 it +should be something like ``long long``. + +Then a more generally review of all the C files in +``rpython/translator/c/src`` for the word ``long``, which means a +32-bit integer even on Win64, replaced it with ``Signed``. + +Then, these two C types have corresponding RPython types: ``rffi.LONG`` +and ``lltype.Signed`` respectively. The first should really correspond +to the C ``long``, as verified by the ``test_rffi_sizeof`` test. The +size of the latter is verified in ``rpython/rlib/rarithmetic``. + +Once these basic tests worked, we reviewed ``rpython/rlib/`` for +uses of ``rffi.LONG`` versus ``lltype.Signed``. The goal was to +fix some more ``LONG-versus-Signed`` issues, by fixing the tests --- as +always run on top of CPython64/64. Note that there was some early work +done in ``rarithmetic`` with the goal of running all the +tests on Win64 on the regular CPython, but this early work was abandoned as a +bad idea. Look only at CPython64/64. + +This was enough to get a translation of PyPy with ``-O2`` +with a minimal set of modules, starting with ``--no-allworkingmodules``; +using CPython64/64 to run this translation too. Careful checking of +the warnings of the C compiler at the end revealed more places that needed +work. By default, MSVC +reports a lot of mismatches of integer sizes as warnings instead of +errors. + +Then we reviewed ``pypy/module/*/`` for ``LONG-versus-Signed`` +issues. This got us a working translated +PyPy on Windows 64 that includes all ``--translationmodules``, i.e. +everything needed to run translations. Once we had that, the hacked +CPython64/64 becomes much less important, because we can run future +translations on top of this translated PyPy. This made it to the nightly +builds on the default branch, and needs to be used by anyone else who wants to +working on Win64. The whole process +ends up with a strange kind of dependency --- we need a translated PyPy in +order to translate a PyPy ---, but that's ok here, as Windows executables are +supposed to never be broken by newer versions of Windows. + +Happy hacking :-) + +.. _`nulano's branch of cpython`: https://github.com/nulano/cpython +.. diff --git a/pypy/goal/pypy.ico b/pypy/goal/pypy.ico Binary files differindex 09d07dcc5a..3dadd59adb 100644 --- a/pypy/goal/pypy.ico +++ b/pypy/goal/pypy.ico diff --git a/pypy/goal/python.manifest b/pypy/goal/python.manifest new file mode 100644 index 0000000000..420ed764bd --- /dev/null +++ b/pypy/goal/python.manifest @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
+ <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
+ <security>
+ <requestedPrivileges>
+ <requestedExecutionLevel level="asInvoker" uiAccess="false"/>
+ </requestedPrivileges>
+ </security>
+ </trustInfo>
+ <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+ <application>
+ <supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
+ <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
+ <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
+ <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
+ <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
+ </application>
+ </compatibility>
+ <application xmlns="urn:schemas-microsoft-com:asm.v3">
+ <windowsSettings>
+ <longPathAware xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">true</longPathAware>
+ </windowsSettings>
+ </application>
+ <dependency>
+ <dependentAssembly>
+ <assemblyIdentity type="win32" name="Microsoft.Windows.Common-Controls"
+ version="6.0.0.0" processorArchitecture="*" publicKeyToken="6595b64144ccf1df" language="*" />
+ </dependentAssembly>
+ </dependency>
+</assembly>
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py index 94a0ac5f28..4c401a7248 100644 --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -1,6 +1,7 @@ import py import os, sys, subprocess +from os.path import join, dirname import pypy from pypy.interpreter import gateway @@ -16,9 +17,9 @@ from pypy.module.thread import os_thread thisdir = py.path.local(__file__).dirpath() try: - this_dir = os.path.dirname(__file__) + this_dir = dirname(__file__) except NameError: - this_dir = os.path.dirname(sys.argv[0]) + this_dir = dirname(sys.argv[0]) def debug(msg): try: @@ -107,7 +108,7 @@ def get_additional_entrypoints(space, w_initstdio): verbose = rffi.cast(lltype.Signed, verbose) if ll_home and ord(ll_home[0]): home1 = rffi.charp2str(ll_home) - home = os.path.join(home1, 'x') # <- so that 'll_home' can be + home = join(home1, 'x') # <- so that 'll_home' can be # directly the root directory else: home1 = "pypy's shared library location" @@ -260,7 +261,8 @@ class PyPyTarget(object): config.translation.suggest(check_str_without_nul=True) config.translation.suggest(shared=True) - config.translation.suggest(icon=os.path.join(this_dir, 'pypy.ico')) + config.translation.suggest(icon=join(this_dir, 'pypy.ico')) + config.translation.suggest(manifest=join(this_dir, 'python.manifest')) if config.translation.shared: if config.translation.output is not None: raise Exception("Cannot use the --output option with PyPy " @@ -350,7 +352,7 @@ class PyPyTarget(object): @taskdef([compile_goal], "Create cffi bindings for modules") def task_build_cffi_imports(self): ''' Use cffi to compile cffi interfaces to modules''' - filename = os.path.join(pypydir, '..', 'lib_pypy', 'pypy_tools', + filename = join(pypydir, '..', 'lib_pypy', 'pypy_tools', 'build_cffi_imports.py') if sys.platform == 'darwin': argv = [filename, '--embed-dependencies'] @@ -381,7 +383,7 @@ class PyPyTarget(object): self.space = make_objspace(config) # manually imports app_main.py - filename = os.path.join(pypydir, 'interpreter', 'app_main.py') + filename = join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False w_dict = app.getwdict(self.space) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py index 6f6879edc1..3ba312c7c8 100755 --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -903,7 +903,21 @@ if __name__ == '__main__': # add an emulator for these pypy-only or 2.7-only functions # (for test_pyc_commandline_argument) - import imp, runpy + try: + import imp, runpy + except ImportError: + if '-S' in sys.argv: + # testing inside a virtualenv and using -S. Add the path of the argv + # file, since the test pre-emptively copied runpy there + from os.path import dirname, exists + tmpdir = dirname(sys.argv[-1]) + if not exists(tmpdir + '/runpy.py'): + tmpdir = dirname(tmpdir) + sys.path.insert(0, tmpdir) + import imp + import runpy + else: + raise def _run_compiled_module(modulename, filename, file, module): import os assert modulename == '__main__' diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py index baa2623b24..6b82d66eb0 100644 --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -347,22 +347,24 @@ class ASTBuilder(object): has_else = False elif_count /= 4 if has_else: + last_elif_node = if_node.get_child(-7) last_elif = if_node.get_child(-6) last_elif_test = self.handle_expr(last_elif) elif_body = self.handle_suite(if_node.get_child(-4)) else_body = self.handle_suite(if_node.get_child(-1)) otherwise = [ast.If(last_elif_test, elif_body, else_body, - last_elif.get_lineno(), last_elif.get_column())] + last_elif_node.get_lineno(), last_elif_node.get_column())] elif_count -= 1 else: otherwise = None for i in range(elif_count): offset = 5 + (elif_count - i - 1) * 4 + elif_node = if_node.get_child(offset - 1) elif_test_node = if_node.get_child(offset) elif_test = self.handle_expr(elif_test_node) elif_body = self.handle_suite(if_node.get_child(offset + 2)) new_if = ast.If(elif_test, elif_body, otherwise, - elif_test_node.get_lineno(), elif_test_node.get_column()) + elif_node.get_lineno(), elif_node.get_column()) otherwise = [new_if] expr = self.handle_expr(if_node.get_child(1)) body = self.handle_suite(if_node.get_child(3)) diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py index 5ec9341dcd..0b53594f19 100644 --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -547,7 +547,7 @@ class PythonCodeGenerator(assemble.PythonCodeMaker): self.pop_frame_block(F_BLOCK_EXCEPT, body) self.emit_jump(ops.JUMP_FORWARD, otherwise) self.use_next_block(exc) - for handler in te.handlers: + for i, handler in enumerate(te.handlers): assert isinstance(handler, ast.ExceptHandler) self.update_position(handler.lineno, True) next_except = self.new_block() @@ -556,6 +556,10 @@ class PythonCodeGenerator(assemble.PythonCodeMaker): handler.type.walkabout(self) self.emit_op_arg(ops.COMPARE_OP, 10) self.emit_jump(ops.POP_JUMP_IF_FALSE, next_except, True) + else: + if i != len(te.handlers) - 1: + self.error( + "bare 'except:' must be the last except block", handler) self.emit_op(ops.POP_TOP) if handler.name: handler.name.walkabout(self) diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py index 91d4ec2b8e..c5508886ae 100644 --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -304,6 +304,20 @@ class TestAstBuilder: assert len(sub_if.orelse) == 1 assert isinstance(sub_if.orelse[0].value, ast.Tuple) + def test_elif_pos_bug(self): + if_ = self.get_first_stmt("if x: 3\nelif \\\n 'hi': pass") + assert isinstance(if_.test, ast.Name) + assert len(if_.orelse) == 1 + sub_if = if_.orelse[0] + assert sub_if.lineno == 2 + assert sub_if.col_offset == 0 + if_ = self.get_first_stmt("if x: 3\nelif \\\n 'hi': pass\nelse: pass") + assert isinstance(if_.test, ast.Name) + assert len(if_.orelse) == 1 + sub_if = if_.orelse[0] + assert sub_if.lineno == 2 + assert sub_if.col_offset == 0 + def test_while(self): wh = self.get_first_stmt("while x: pass") assert isinstance(wh, ast.While) diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py index f2631ae620..f7c779835b 100644 --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -770,6 +770,18 @@ class TestCompiler(BaseTestCompiler): """ py.test.raises(SyntaxError, self.simple_test, source, None, None) + def test_bare_except_not_last(self): + source = """if 1: + try: + pass + except: + pass + except ValueError: + pass + """ + with py.test.raises(SyntaxError): + self.simple_test(source, None, None) + def test_unpack_singletuple(self): source = """if 1: l = [] diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py index 6a7f3a5339..ef524b3927 100644 --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -704,6 +704,10 @@ class ObjSpace(object): # is required to initialise sys on Windows. from pypy.module.cpyext.state import State self.fromcache(State).build_api() + elif self.config.objspace.usemodules._cffi_backend: + from pypy.module._cffi_backend import copy_includes + copy_includes.main() + self.getbuiltinmodule('sys') self.getbuiltinmodule('imp') self.getbuiltinmodule('__builtin__') diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py index dfbd2a5dd2..94f18e6c86 100644 --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1320,7 +1320,7 @@ class __extend__(pyframe.PyFrame): for i in range(itemcount-1, -1, -1): w_item = self.peekvalue(i) self.space.call_method(w_set, 'add', w_item) - self.popvalues(itemcount) + self.dropvalues(itemcount) self.pushvalue(w_set) def STORE_MAP(self, oparg, next_instr): diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py index 5490366510..80476526f7 100644 --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -23,6 +23,18 @@ def vars(*obj): except AttributeError: raise TypeError("vars() argument must have __dict__ attribute") +# These are defined in the types module, but we cannot always import it. +# virtualenv when run with -S for instance. Instead, copy the code to create +# the needed types to be checked. +class types(object): + class _C: + def _m(self): pass + ModuleType = type(sys) + ClassType = type(_C) + TypeType = type + _x = _C() + InstanceType = type(_x) + def dir(*args): """dir([object]) -> list of strings @@ -45,7 +57,7 @@ def dir(*args): local_names.sort() return local_names - import types + # import types obj = args[0] if isinstance(obj, types.InstanceType): dir_meth = getattr(obj, '__dir__', None) @@ -58,6 +70,8 @@ def dir(*args): type(names),)) names.sort() return names + # From here, this is python2-specific since in python3 + # everything has a __dir__ elif isinstance(obj, types.ModuleType): try: return sorted(obj.__dict__) diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py index c3b7773b3e..cf7a0af13c 100644 --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -10,6 +10,7 @@ from pypy.objspace.std.setobject import W_BaseSetObject from pypy.objspace.std.typeobject import MethodCache from pypy.objspace.std.mapdict import MapAttrCache from rpython.rlib import rposix, rgc, rstack +from rpython.rtyper.lltypesystem import rffi def internal_repr(space, w_object): @@ -118,6 +119,19 @@ def get_console_cp(space): space.newtext('cp%d' % rwin32.GetConsoleOutputCP()), ]) +@unwrap_spec(fd=int) +def get_osfhandle(space, fd): + """get_osfhandle() + + Return the handle corresponding to the file descriptor (windows only) + """ + from rpython.rlib import rwin32 # Windows only + try: + ret = rwin32.get_osfhandle(fd) + return space.newint(rffi.cast(rffi.INT, ret)) + except OSError as e: + raise wrap_oserror(space, e) + @unwrap_spec(sizehint=int) def resizelist_hint(space, w_list, sizehint): """ Reallocate the underlying storage of the argument list to sizehint """ diff --git a/pypy/module/__pypy__/moduledef.py b/pypy/module/__pypy__/moduledef.py index 22491d42ca..8a608d2868 100644 --- a/pypy/module/__pypy__/moduledef.py +++ b/pypy/module/__pypy__/moduledef.py @@ -122,6 +122,7 @@ class Module(MixedModule): } if sys.platform == 'win32': interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp' + interpleveldefs['get_osfhandle'] = 'interp_magic.get_osfhandle' submodules = { "builders": BuildersModule, diff --git a/pypy/module/__pypy__/test/test_magic.py b/pypy/module/__pypy__/test/test_magic.py index e6a6a58e5f..c79b2c196f 100644 --- a/pypy/module/__pypy__/test/test_magic.py +++ b/pypy/module/__pypy__/test/test_magic.py @@ -1,8 +1,13 @@ # encoding: utf-8 +import pytest +import sys class AppTestMagic: spaceconfig = dict(usemodules=['__pypy__']) + def setup_class(cls): + cls.w_file = cls.space.wrap(__file__) + def test_save_module_content_for_future_reload(self): import sys, __pypy__ d = sys.dont_write_bytecode @@ -66,3 +71,10 @@ def f(): from __pypy__ import utf8content assert utf8content(u"a") == b"a" assert utf8content(u"\xe4") == b'\xc3\xa4' + + @pytest.mark.skipif(sys.platform != 'win32', reason="win32 only") + def test_get_osfhandle(self): + from __pypy__ import get_osfhandle + with open(self.file) as fid: + f = get_osfhandle(fid.fileno()) + raises(OSError, get_osfhandle, 2**30) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py index b78c38e9c9..43fc4a2ac4 100644 --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1 +1 @@ -VERSION = "1.14.3" +VERSION = "1.14.4" diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py index c4dac6c9ef..b056c6b7cc 100644 --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -676,6 +676,14 @@ class W_CDataFromBuffer(W_CData): def get_array_length(self): return self.length + def _sizeof(self): + from pypy.module._cffi_backend import ctypearray + ctype = self.ctype + if isinstance(ctype, ctypearray.W_CTypeArray): + return self.length * ctype.ctitem.size + else: + return W_CData._sizeof(self) + def _repr_extra(self): from pypy.module._cffi_backend import ctypearray if self.w_keepalive is None: diff --git a/pypy/module/_cffi_backend/copy_includes.py b/pypy/module/_cffi_backend/copy_includes.py new file mode 100644 index 0000000000..a4415ba0fb --- /dev/null +++ b/pypy/module/_cffi_backend/copy_includes.py @@ -0,0 +1,64 @@ +import shutil +import textwrap + +from os.path import dirname, join, exists + +include = join(dirname(__file__), '..', '..', '..', 'include') +assert exists(include) +cpyext_include = join(dirname(__file__), '..', 'cpyext', 'include') + +def main(): + """Copy/create just enough header information to allow cffi to compile c-extension modules + """ + python_h = textwrap.dedent(""" + /* Partial C-API headers to allow CFFI C-compiled modules to work with PyPy */ + #include <sys/types.h> + #include <stdarg.h> + + #ifdef __GNUC__ + #define _GNU_SOURCE 1 + #endif + #ifndef _WIN32 + # define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__)) + # define PyAPI_FUNC(RTYPE) __attribute__((visibility("default"))) RTYPE + # define PyAPI_DATA(RTYPE) extern PyAPI_FUNC(RTYPE) + # define Py_LOCAL_INLINE(type) static inline type + #else + # define Py_DEPRECATED(VERSION_UNUSED) + # define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE + # define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE + # define Py_LOCAL_INLINE(type) static __inline type __fastcall + #endif + + typedef void PyObject; + /* CPython sets Py_ssize_t in pyport.h, PyPy in cpyext_object.h */ + #ifdef _WIN64 + typedef long long Py_ssize_t; + #else + typedef long Py_ssize_t; + #endif + + #include <patchlevel.h> + #include <modsupport.h> + + #include <stdarg.h> + #include <stdio.h> + #include <string.h> + #include <assert.h> + #include <locale.h> + #include <ctype.h> + + /* normally defined in "pythread.h", but we can't include that */ + #define WITH_THREAD + """) + if exists(join(include, 'Python.h')): + return + with open(join(include, 'Python.h'), 'wt') as fid: + fid.write(python_h) + for header in ('patchlevel.h', 'modsupport.h', 'pyconfig.h'): + shutil.copy(join(cpyext_include, header), join(include, header)) + + +if __name__ == '__main__': + main() + diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py index 07d291a0b7..5e7a4db2f1 100644 --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -275,10 +275,10 @@ class W_CTypePrimitiveSigned(W_CTypePrimitive): w_cdata.write_raw_signed_data(value) def unpack_list_of_int_items(self, ptr, length): - if self.size == rffi.sizeof(rffi.LONG): + if self.size == rffi.sizeof(rffi.SIGNED): from rpython.rlib.rrawarray import populate_list_from_raw_array res = [] - buf = rffi.cast(rffi.LONGP, ptr) + buf = rffi.cast(rffi.SIGNEDP, ptr) populate_list_from_raw_array(res, buf, length) return res elif self.value_smaller_than_long: @@ -291,9 +291,9 @@ class W_CTypePrimitiveSigned(W_CTypePrimitive): int_list = self.space.listview_int(w_ob) if (int_list is not None and self._within_bounds(len(int_list), expected_length)): - if self.size == rffi.sizeof(rffi.LONG): # fastest path + if self.size == rffi.sizeof(rffi.SIGNED): # fastest path from rpython.rlib.rrawarray import copy_list_to_raw_array - cdata = rffi.cast(rffi.LONGP, cdata) + cdata = rffi.cast(rffi.SIGNEDP, cdata) copy_list_to_raw_array(int_list, cdata) else: overflowed = misc.pack_list_to_raw_array_bounds_signed( diff --git a/pypy/module/_cffi_backend/ffi_obj.py b/pypy/module/_cffi_backend/ffi_obj.py index 0da57ea5e5..1cf5869b55 100644 --- a/pypy/module/_cffi_backend/ffi_obj.py +++ b/pypy/module/_cffi_backend/ffi_obj.py @@ -558,13 +558,14 @@ Return the size in bytes of the argument. It can be a string naming a C type, or a 'cdata' instance.""" # if isinstance(w_arg, W_CData): + w_ctype = w_arg.ctype size = w_arg._sizeof() else: w_ctype = self.ffi_type(w_arg, ACCEPT_ALL) size = w_ctype.size - if size < 0: - raise oefmt(self.w_FFIError, - "don't know the size of ctype '%s'", w_ctype.name) + if size < 0: + raise oefmt(self.w_FFIError, + "don't know the size of ctype '%s'", w_ctype.name) return self.space.newint(size) diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py index 239692605f..04dca73bf5 100644 --- a/pypy/module/_cffi_backend/func.py +++ b/pypy/module/_cffi_backend/func.py @@ -39,13 +39,15 @@ def typeof(space, w_cdata): def sizeof(space, w_obj): if isinstance(w_obj, cdataobj.W_CData): size = w_obj._sizeof() + ctype = w_obj.ctype elif isinstance(w_obj, ctypeobj.W_CType): size = w_obj.size - if size < 0: - raise oefmt(space.w_ValueError, - "ctype '%s' is of unknown size", w_obj.name) + ctype = w_obj else: raise oefmt(space.w_TypeError, "expected a 'cdata' or 'ctype' object") + if size < 0: + raise oefmt(space.w_ValueError, + "ctype '%s' is of unknown size", ctype.name) return space.newint(size) @unwrap_spec(w_ctype=ctypeobj.W_CType) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py index 3cc6ae8fb7..cd1e648917 100644 --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.14.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.14.4", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): @@ -3958,6 +3958,20 @@ def test_from_buffer_types(): with pytest.raises(ValueError): release(pv[0]) +def test_issue483(): + BInt = new_primitive_type("int") + BIntP = new_pointer_type(BInt) + BIntA = new_array_type(BIntP, None) + lst = list(range(25)) + bytestring = bytearray(buffer(newp(BIntA, lst))[:] + b'XYZ') + p1 = from_buffer(BIntA, bytestring) # int[] + assert len(buffer(p1)) == 25 * size_of_int() + assert sizeof(p1) == 25 * size_of_int() + # + p2 = from_buffer(BIntP, bytestring) + assert sizeof(p2) == size_of_ptr() + assert len(buffer(p2)) == size_of_int() # first element only, by default + def test_memmove(): Short = new_primitive_type("short") ShortA = new_array_type(new_pointer_type(Short), None) diff --git a/pypy/module/_cffi_backend/test/test_fastpath.py b/pypy/module/_cffi_backend/test/test_fastpath.py index c5b35a8da4..bf6c896ad5 100644 --- a/pypy/module/_cffi_backend/test/test_fastpath.py +++ b/pypy/module/_cffi_backend/test/test_fastpath.py @@ -75,15 +75,19 @@ class AppTest_fast_path_from_list(object): def test_fast_init_ulong_from_list(self): import sys import _cffi_backend + maxlong = sys.maxint + if sys.platform == 'win32': + # maxlong == 2**31-1 < sys.maxint == 2**63-1 on win64! + maxlong = int(2**31-1) ULONG = _cffi_backend.new_primitive_type('unsigned long') P_ULONG = _cffi_backend.new_pointer_type(ULONG) ULONG_ARRAY = _cffi_backend.new_array_type(P_ULONG, None) - buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, sys.maxint]) + buf = _cffi_backend.newp(ULONG_ARRAY, [1, 2, maxlong]) assert buf[0] == 1 assert buf[1] == 2 - assert buf[2] == sys.maxint + assert buf[2] == maxlong raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-1]) - raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-sys.maxint]) + raises(OverflowError, _cffi_backend.newp, ULONG_ARRAY, [-maxlong]) def test_fast_init_cfloat_from_list(self): import _cffi_backend diff --git a/pypy/module/_cppyy/test/conftest.py b/pypy/module/_cppyy/test/conftest.py index 44e056de9a..86c9e7aa1f 100644 --- a/pypy/module/_cppyy/test/conftest.py +++ b/pypy/module/_cppyy/test/conftest.py @@ -38,10 +38,13 @@ def pytest_ignore_collect(path, config): return True disabled = None +if sys.maxsize > 2**32 and sys.platform == 'win32': + # cppyy not yet supported on windows 64 bit + disabled = True def pytest_configure(config): global disabled - if config.getoption('runappdirect') or config.getoption('direct_apptest'): + if disabled or config.getoption('runappdirect') or config.getoption('direct_apptest'): if py.path.local.sysfind('genreflex') is None: disabled = True # can't run dummy tests in -A return diff --git a/pypy/module/_demo/demo.py b/pypy/module/_demo/demo.py index a5b6e3b02b..316321d8a3 100644 --- a/pypy/module/_demo/demo.py +++ b/pypy/module/_demo/demo.py @@ -7,7 +7,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo import math -time_t = rffi_platform.getsimpletype('time_t', '#include <time.h>', rffi.LONG) +time_t = rffi_platform.getsimpletype('time_t', '#include <time.h>', rffi.SIGNED) eci = ExternalCompilationInfo(includes=['time.h']) time = rffi.llexternal('time', [lltype.Signed], time_t, diff --git a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h index 164442e950..fce575835e 100644 --- a/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h +++ b/pypy/module/_multibytecodec/src/cjkcodecs/multibytecodec.h @@ -9,7 +9,7 @@ #include <assert.h> #ifdef _WIN64 -typedef __int64 pypymbc_ssize_t +typedef __int64 pypymbc_ssize_t; #elif defined(_WIN32) typedef int pypymbc_ssize_t; #else diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py index f39b01203d..9772b8056a 100644 --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -2,7 +2,7 @@ import sys from errno import EINTR from rpython.rlib import rpoll, rsocket -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.baseobjspace import W_Root @@ -446,7 +446,7 @@ class W_PipeConnection(W_BaseConnection): left_ptr): raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) - length = intmask(read_ptr[0] + left_ptr[0]) + length = intmask(read_ptr[0]) + intmask(left_ptr[0]) if length > maxlength: # bad message, close connection self.flags &= ~READABLE if self.flags == 0: @@ -454,17 +454,18 @@ class W_PipeConnection(W_BaseConnection): raise oefmt(space.w_IOError, "bad message length") newbuf = lltype.malloc(rffi.CCHARP.TO, length + 1, flavor='raw') - for i in range(read_ptr[0]): + length_read = intmask(read_ptr[0]) + for i in range(length_read): newbuf[i] = self.buffer[i] result = _ReadFile(self.handle, - rffi.ptradd(newbuf, read_ptr[0]), left_ptr[0], + rffi.ptradd(newbuf, length_read), left_ptr[0], read_ptr, rffi.NULL) if not result: rffi.free_charp(newbuf) raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) - assert read_ptr[0] == left_ptr[0] + assert r_uint(read_ptr[0]) == r_uint(left_ptr[0]) return length, newbuf finally: lltype.free(read_ptr, flavor='raw') @@ -483,7 +484,7 @@ class W_PipeConnection(W_BaseConnection): bytes_ptr, lltype.nullptr(rwin32.LPDWORD.TO)): raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) - bytes = bytes_ptr[0] + bytes = r_uint(bytes_ptr[0]) finally: lltype.free(bytes_ptr, flavor='raw') @@ -510,7 +511,7 @@ class W_PipeConnection(W_BaseConnection): lltype.nullptr(rwin32.LPDWORD.TO)): raise wrap_windowserror(space, rwin32.lastSavedWindowsError()) - bytes = bytes_ptr[0] + bytes = r_uint(bytes_ptr[0]) finally: lltype.free(bytes_ptr, flavor='raw') diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py index 2ae83028af..1426bd6925 100644 --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -4,7 +4,7 @@ import sys import time from rpython.rlib import jit, rgc, rthread -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform as platform from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -22,7 +22,7 @@ if sys.platform == 'win32': from pypy.module._multiprocessing.interp_win32 import ( _GetTickCount, handle_w) - SEM_VALUE_MAX = sys.maxint + SEM_VALUE_MAX = int(2**31-1) # max rffi.LONG _CreateSemaphore = rwin32.winexternal( 'CreateSemaphoreA', [rffi.VOIDP, rffi.LONG, rffi.LONG, rwin32.LPCSTR], @@ -316,7 +316,7 @@ if sys.platform == 'win32': try: if not _ReleaseSemaphore(self.handle, 1, previous_ptr): raise rwin32.lastSavedWindowsError("ReleaseSemaphore") - return previous_ptr[0] + 1 + return intmask(previous_ptr[0]) + 1 finally: lltype.free(previous_ptr, flavor='raw') diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py index b366f3c9b4..8d9a96b953 100644 --- a/pypy/module/_multiprocessing/interp_win32.py +++ b/pypy/module/_multiprocessing/interp_win32.py @@ -147,13 +147,13 @@ def SetNamedPipeHandleState(space, w_handle, w_pipemode, w_maxinstances, zero=True) try: if not space.is_w(w_pipemode, space.w_None): - state[0] = space.uint_w(w_pipemode) + state[0] = rffi.cast(rffi.UINT, space.uint_w(w_pipemode)) statep[0] = rffi.ptradd(state, 0) if not space.is_w(w_maxinstances, space.w_None): - state[1] = space.uint_w(w_maxinstances) + state[1] = rffi.cast(rffi.UINT, space.uint_w(w_maxinstances)) statep[1] = rffi.ptradd(state, 1) if not space.is_w(w_timeout, space.w_None): - state[2] = space.uint_w(w_timeout) + state[2] = rffi.cast(rffi.UINT, space.uint_w(w_timeout)) statep[2] = rffi.ptradd(state, 2) if not _SetNamedPipeHandleState(handle, statep[0], statep[1], statep[2]): diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py b/pypy/module/_multiprocessing/test/test_semaphore.py index 963be9ffa3..71e5098bd1 100644 --- a/pypy/module/_multiprocessing/test/test_semaphore.py +++ b/pypy/module/_multiprocessing/test/test_semaphore.py @@ -84,6 +84,31 @@ class AppTestSemaphore: sem.release() sem.release() + def test_semaphore_maxvalue(self): + from _multiprocessing import SemLock + import sys + kind = self.SEMAPHORE + value = SemLock.SEM_VALUE_MAX + maxvalue = SemLock.SEM_VALUE_MAX + sem = SemLock(kind, value, maxvalue) + + for i in range(10): + res = sem.acquire() + assert res == True + assert sem._count() == i+1 + if sys.platform != 'darwin': + assert sem._get_value() == maxvalue - (i+1) + + value = 0 + maxvalue = SemLock.SEM_VALUE_MAX + sem = SemLock(kind, value, maxvalue) + + for i in range(10): + sem.release() + assert sem._count() == -(i+1) + if sys.platform != 'darwin': + assert sem._get_value() == i+1 + def test_semaphore_wait(self): from _multiprocessing import SemLock kind = self.SEMAPHORE diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py index cae51b99b3..8dac984733 100644 --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -238,8 +238,8 @@ class JSONDecoder(W_Root): def decode_float(self, i): from rpython.rlib import rdtoa start = rffi.ptradd(self.ll_chars, i) - floatval = rdtoa.dg_strtod(start, self.end_ptr) - diff = rffi.cast(rffi.LONG, self.end_ptr[0]) - rffi.cast(rffi.LONG, start) + floatval = rdtoa.dg_strtod(rffi.cast(rffi.CONST_CCHARP, start), self.end_ptr) + diff = rffi.cast(rffi.SIGNED, self.end_ptr[0]) - rffi.cast(rffi.SIGNED, start) self.pos = i + diff return self.space.newfloat(floatval) diff --git a/pypy/module/_pypyjson/simd.py b/pypy/module/_pypyjson/simd.py index 8a318c7d66..586a222962 100644 --- a/pypy/module/_pypyjson/simd.py +++ b/pypy/module/_pypyjson/simd.py @@ -95,7 +95,7 @@ def find_end_of_string_simd_unaligned(ll_chars, startpos, length): ch = ll_chars[startpos] strhash = (ord(ch) << 7) ^ 0x345678 - wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + wordarray = rffi.cast(rffi.UNSIGNEDP, rffi.ptradd(ll_chars, startpos)) num_safe_reads = (length - startpos) // WORD_SIZE bits = 0 @@ -142,7 +142,7 @@ def find_end_of_string_simd_unaligned(ll_chars, startpos, length): def find_end_of_string_simd_unaligned_no_hash(ll_chars, startpos, length): ch = ll_chars[startpos] - wordarray = rffi.cast(rffi.ULONGP, rffi.ptradd(ll_chars, startpos)) + wordarray = rffi.cast(rffi.UNSIGNEDP, rffi.ptradd(ll_chars, startpos)) num_safe_reads = (length - startpos) // WORD_SIZE bits = 0 diff --git a/pypy/module/_pypyjson/test/test_simd.py b/pypy/module/_pypyjson/test/test_simd.py index ee512b05e8..b293b74aa3 100644 --- a/pypy/module/_pypyjson/test/test_simd.py +++ b/pypy/module/_pypyjson/test/test_simd.py @@ -27,7 +27,7 @@ def string_to_word(s): assert len(s) == WORD_SIZE ll_chars, llobj, flag = rffi.get_nonmovingbuffer_ll_final_null(s) try: - wordarray = rffi.cast(rffi.ULONGP, ll_chars) + wordarray = rffi.cast(rffi.UNSIGNEDP, ll_chars) return wordarray[0] finally: rffi.free_nonmovingbuffer_ll(ll_chars, llobj, flag) diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py index 89fe4bdd94..2e5483e267 100644 --- a/pypy/module/_rawffi/alt/interp_funcptr.py +++ b/pypy/module/_rawffi/alt/interp_funcptr.py @@ -129,7 +129,7 @@ class W_FuncPtr(W_Root): """ Return the physical address in memory of the function """ - return space.newint(rffi.cast(rffi.LONG, self.func.funcsym)) + return space.newint(rffi.cast(rffi.SIGNED, self.func.funcsym)) class PushArgumentConverter(FromAppLevelConverter): @@ -164,13 +164,13 @@ class PushArgumentConverter(FromAppLevelConverter): def handle_char_p(self, w_ffitype, w_obj, strval): buf = rffi.str2charp(strval) self.w_func.to_free.append(rffi.cast(rffi.VOIDP, buf)) - addr = rffi.cast(rffi.ULONG, buf) + addr = rffi.cast(lltype.Unsigned, buf) self.argchain.arg(addr) def handle_unichar_p(self, w_ffitype, w_obj, utf8val, utf8len): buf = rffi.utf82wcharp(utf8val, utf8len) self.w_func.to_free.append(rffi.cast(rffi.VOIDP, buf)) - addr = rffi.cast(rffi.ULONG, buf) + addr = rffi.cast(lltype.Unsigned, buf) self.argchain.arg(addr) def handle_float(self, w_ffitype, w_obj, floatval): @@ -212,39 +212,48 @@ class CallFunctionConverter(ToAppLevelConverter): # the result buffer may contains garbage in its higher bits. To get # the correct value, and to be sure to handle the signed/unsigned case # correctly, we need to cast the result to the correct type. After - # that, we cast it back to LONG, because this is what we want to pass + # that, we cast it back to SIGNED, because this is what we want to pass # to space.newint in order to get a nice applevel <int>. # restype = w_ffitype.get_ffitype() call = self.func.call if restype is libffi.types.slong: - x = call(self.argchain, rffi.LONG) + # cast required for 64-bit Windows + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.LONG)) + elif restype is libffi.types.slonglong: + assert libffi.IS_WIN64 + # for 64-bit Windows, handled with slong above on other systems + x = call(self.argchain, rffi.LONGLONG) elif restype is libffi.types.sint: - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.INT)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.INT)) elif restype is libffi.types.sshort: - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SHORT)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.SHORT)) elif restype is libffi.types.schar: - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.SIGNEDCHAR)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.SIGNEDCHAR)) else: raise self.error(w_ffitype) return x def get_unsigned(self, w_ffitype): - return self.func.call(self.argchain, rffi.ULONG) + return self.func.call(self.argchain, lltype.Unsigned) def get_unsigned_which_fits_into_a_signed(self, w_ffitype): # the same comment as get_signed apply restype = w_ffitype.get_ffitype() call = self.func.call - if restype is libffi.types.uint: + if restype is libffi.types.ulong: + assert libffi.IS_WIN64 + # on win64 this is the same as uint on 32-bit below + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.ULONG)) + elif restype is libffi.types.uint: assert not libffi.IS_32_BIT # on 32bit machines, we should never get here, because it's a case # which has already been handled by get_unsigned above. - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UINT)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.UINT)) elif restype is libffi.types.ushort: - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.USHORT)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.USHORT)) elif restype is libffi.types.uchar: - x = rffi.cast(rffi.LONG, call(self.argchain, rffi.UCHAR)) + x = rffi.cast(rffi.SIGNED, call(self.argchain, rffi.UCHAR)) else: raise self.error(w_ffitype) return x @@ -252,7 +261,7 @@ class CallFunctionConverter(ToAppLevelConverter): def get_pointer(self, w_ffitype): ptrres = self.func.call(self.argchain, rffi.VOIDP) - return rffi.cast(rffi.ULONG, ptrres) + return rffi.cast(lltype.Unsigned, ptrres) def get_char(self, w_ffitype): return self.func.call(self.argchain, rffi.UCHAR) @@ -267,11 +276,11 @@ class CallFunctionConverter(ToAppLevelConverter): return self.func.call(self.argchain, rffi.FLOAT) def get_struct(self, w_ffitype, w_structdescr): - addr = self.func.call(self.argchain, rffi.LONG, is_struct=True) + addr = self.func.call(self.argchain, rffi.SIGNED, is_struct=True) return w_structdescr.fromaddress(self.space, addr) def get_struct_rawffi(self, w_ffitype, w_structdescr): - uintval = self.func.call(self.argchain, rffi.ULONG, is_struct=True) + uintval = self.func.call(self.argchain, lltype.Unsigned, is_struct=True) return w_structdescr.fromaddress(self.space, uintval) def get_void(self, w_ffitype): diff --git a/pypy/module/_rawffi/alt/interp_struct.py b/pypy/module/_rawffi/alt/interp_struct.py index ead62b23a1..b93d753639 100644 --- a/pypy/module/_rawffi/alt/interp_struct.py +++ b/pypy/module/_rawffi/alt/interp_struct.py @@ -177,7 +177,7 @@ class W__StructInstance(W_Root): self.rawmem = lltype.nullptr(rffi.VOIDP.TO) def getaddr(self, space): - addr = rffi.cast(rffi.ULONG, self.rawmem) + addr = rffi.cast(lltype.Unsigned, self.rawmem) return space.newint(addr) @unwrap_spec(name='text') diff --git a/pypy/module/_rawffi/alt/test/test_funcptr.py b/pypy/module/_rawffi/alt/test/test_funcptr.py index 6451872e6c..6aa2543576 100644 --- a/pypy/module/_rawffi/alt/test/test_funcptr.py +++ b/pypy/module/_rawffi/alt/test/test_funcptr.py @@ -40,13 +40,15 @@ class BaseAppTestFFI(object): def setup_class(cls): space = cls.space cls.w_iswin32 = space.wrap(sys.platform == 'win32') + cls.w_iswin64 = space.wrap(sys.platform == 'win32' + and sys.maxint == 2**63-1) cls.w_libfoo_name = space.wrap(cls.prepare_c_example()) cls.w_libc_name = space.wrap(get_libc_name()) libm_name = get_libm_name(sys.platform) cls.w_libm_name = space.wrap(libm_name) libm = CDLL(libm_name) pow = libm.getpointer('pow', [], types.void) - pow_addr = rffi.cast(rffi.LONG, pow.funcsym) + pow_addr = rffi.cast(rffi.SIGNED, pow.funcsym) cls._libm = libm # otherwise it gets unloaded - argh! cls.w_pow_addr = space.wrap(pow_addr) @@ -302,13 +304,16 @@ class AppTestFFI(BaseAppTestFFI): """ import sys from _rawffi.alt import CDLL, types + maxlong = sys.maxint + if sys.platform == 'win32': + maxlong = 2147483647 libfoo = CDLL(self.libfoo_name) sum_xy = libfoo.getfunc('sum_xy_ul', [types.ulong, types.ulong], types.ulong) - assert sum_xy(sys.maxint, 12) == sys.maxint+12 - assert sum_xy(sys.maxint+1, 12) == sys.maxint+13 + assert sum_xy(maxlong, 12) == maxlong+12 + assert sum_xy(maxlong+1, 12) == maxlong+13 # - res = sum_xy(sys.maxint*2+3, 0) + res = sum_xy(maxlong*2+3, 0) assert res == 1 def test_unsigned_short_args(self): @@ -574,8 +579,9 @@ class AppTestFFI(BaseAppTestFFI): raises(AttributeError, "libnone.getfunc('I_do_not_exist', [], types.void)") def test_calling_convention1(self): - if not self.iswin32: - skip("windows specific") + # win64 doesn't have __stdcall + if not self.iswin32 or self.iswin64: + skip("windows 32-bit specific") from _rawffi.alt import WinDLL, types libm = WinDLL(self.libm_name) pow = libm.getfunc('pow', [types.double, types.double], types.double) @@ -595,8 +601,9 @@ class AppTestFFI(BaseAppTestFFI): sleep(10) def test_calling_convention3(self): - if not self.iswin32: - skip("windows specific") + # win64 doesn't have __stdcall + if not self.iswin32 or self.iswin64: + skip("windows 32-bit specific") from _rawffi.alt import CDLL, types wrong_kernel = CDLL('Kernel32.dll') wrong_sleep = wrong_kernel.getfunc('Sleep', [types.uint], types.void) @@ -608,8 +615,9 @@ class AppTestFFI(BaseAppTestFFI): assert 0, 'test must assert, wrong calling convention' def test_func_fromaddr2(self): - if not self.iswin32: - skip("windows specific") + # win64 doesn't have __stdcall + if not self.iswin32 or self.iswin64: + skip("windows 32-bit specific") from _rawffi.alt import CDLL, types, FuncPtr from _rawffi import FUNCFLAG_STDCALL libm = CDLL(self.libm_name) @@ -624,6 +632,7 @@ class AppTestFFI(BaseAppTestFFI): assert 0, 'test must assert, wrong calling convention' def test_func_fromaddr3(self): + # win64: check FUNCFLAG_STDCALL is ignored on win64, as it should be if not self.iswin32: skip("windows specific") from _rawffi.alt import WinDLL, types, FuncPtr diff --git a/pypy/module/_rawffi/alt/test/test_struct.py b/pypy/module/_rawffi/alt/test/test_struct.py index f3438eba41..bf933c362b 100644 --- a/pypy/module/_rawffi/alt/test/test_struct.py +++ b/pypy/module/_rawffi/alt/test/test_struct.py @@ -135,7 +135,9 @@ class AppTestStruct(BaseAppTestFFI): def test_getfield_setfield_signed_types(self): import sys from _rawffi.alt import _StructDescr, Field, types - longsize = types.slong.sizeof() + maxlong = sys.maxint + if sys.platform == 'win32': + maxlong = 2147483647 fields = [ Field('sbyte', types.sbyte), Field('sshort', types.sshort), @@ -150,15 +152,17 @@ class AppTestStruct(BaseAppTestFFI): assert struct.getfield('sshort') == -32768 struct.setfield('sint', 43) assert struct.getfield('sint') == 43 - struct.setfield('slong', sys.maxint+1) - assert struct.getfield('slong') == -sys.maxint-1 - struct.setfield('slong', sys.maxint*3) - assert struct.getfield('slong') == sys.maxint-2 + struct.setfield('slong', maxlong+1) + assert struct.getfield('slong') == -maxlong-1 + struct.setfield('slong', maxlong*3) + assert struct.getfield('slong') == maxlong-2 def test_getfield_setfield_unsigned_types(self): import sys from _rawffi.alt import _StructDescr, Field, types - longsize = types.slong.sizeof() + maxlong = sys.maxint + if sys.platform == 'win32': + maxlong = 2147483647 fields = [ Field('ubyte', types.ubyte), Field('ushort', types.ushort), @@ -177,8 +181,8 @@ class AppTestStruct(BaseAppTestFFI): struct.setfield('uint', 43) assert struct.getfield('uint') == 43 struct.setfield('ulong', -1) - assert struct.getfield('ulong') == sys.maxint*2 + 1 - struct.setfield('ulong', sys.maxint*2 + 2) + assert struct.getfield('ulong') == maxlong*2 + 1 + struct.setfield('ulong', maxlong*2 + 2) assert struct.getfield('ulong') == 0 struct.setfield('char', b'a') assert struct.getfield('char') == b'a' diff --git a/pypy/module/_rawffi/alt/type_converter.py b/pypy/module/_rawffi/alt/type_converter.py index 0de4c6ee09..4344e6e004 100644 --- a/pypy/module/_rawffi/alt/type_converter.py +++ b/pypy/module/_rawffi/alt/type_converter.py @@ -204,16 +204,16 @@ class ToAppLevelConverter(object): intval = self.get_signed(w_ffitype) return space.newint(intval) elif (w_ffitype is app_types.ulonglong or - w_ffitype is app_types.ulong or (libffi.IS_32_BIT and - w_ffitype is app_types.uint)): + (not libffi.IS_WIN64 and w_ffitype is app_types.ulong) or + (libffi.IS_32_BIT and w_ffitype is app_types.uint)): # Note that we the second check (for ulonglong) is meaningful only # on 64 bit, because on 32 bit the ulonglong case would have been # handled by the is_longlong() branch above. On 64 bit, ulonglong - # is essentially the same as ulong. + # is essentially the same as ulong unless we are on win64. # - # We need to be careful when the return type is ULONG, because the - # value might not fit into a signed LONG, and thus might require - # and app-evel <long>. This is why we need to treat it separately + # We need to be careful when the return type is ULONGLONG, because + # the value might not fit into a SIGNED, and thus might require + # an app-level <long>. This is why we need to treat it separately # than the other unsigned types. uintval = self.get_unsigned(w_ffitype) return space.newint(uintval) diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py index 03a76f74bc..e44327450d 100644 --- a/pypy/module/_rawffi/callback.py +++ b/pypy/module/_rawffi/callback.py @@ -45,7 +45,7 @@ def callback(ll_args, ll_res, ll_userdata): space, rffi.cast(rffi.SIZE_T, ll_args[i])) else: # XXX other types? - args_w[i] = space.newint(rffi.cast(rffi.ULONG, ll_args[i])) + args_w[i] = space.newint(rffi.cast(lltype.Unsigned, ll_args[i])) w_res = space.call(w_callable, space.newtuple(args_w)) if callback_ptr.result is not None: # don't return void ptr = ll_res diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py index 960b3e075c..88e73532b0 100644 --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -245,6 +245,7 @@ class AppTestFfi: cls.w_sizes_and_alignments = space.wrap(dict( [(k, (v.c_size, v.c_alignment)) for k,v in TYPEMAP.iteritems()])) cls.w_float_typemap = space.wrap(TYPEMAP_FLOAT_LETTERS) + cls.w_is64bit = space.wrap(sys.maxint > 2147483647) def test_libload(self): import _rawffi @@ -719,7 +720,7 @@ class AppTestFfi: ll_to_sort = _rawffi.Array('i')(4) for i in range(4): ll_to_sort[i] = 4-i - qsort = libc.ptr('qsort', ['P', 'l', 'l', 'P'], None) + qsort = libc.ptr('qsort', ['P', 'Z', 'Z', 'P'], None) bogus_args = [] def compare(a, b): a1 = _rawffi.Array('i').fromaddress(_rawffi.Array('P').fromaddress(a, 1)[0], 1) @@ -730,11 +731,11 @@ class AppTestFfi: return 1 return -1 a1 = ll_to_sort.byptr() - a2 = _rawffi.Array('l')(1) + a2 = _rawffi.Array('Z')(1) a2[0] = len(ll_to_sort) - a3 = _rawffi.Array('l')(1) + a3 = _rawffi.Array('Z')(1) a3[0] = struct.calcsize('i') - cb = _rawffi.CallbackPtr(compare, ['P', 'P'], 'l') + cb = _rawffi.CallbackPtr(compare, ['P', 'P'], 'i') a4 = cb.byptr() qsort(a1, a2, a3, a4) res = [ll_to_sort[i] for i in range(len(ll_to_sort))] @@ -1022,8 +1023,8 @@ class AppTestFfi: raises(_rawffi.SegfaultException, a.__setitem__, 3, 3) def test_stackcheck(self): - if self.platform != "msvc": - skip("win32 msvc specific") + if self.platform != "msvc" or self.is64bit: + skip("32-bit win32 msvc specific") # Even if the call corresponds to the specified signature, # the STDCALL calling convention may detect some errors diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py index 66124e29e1..1b96c1b67e 100644 --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -550,13 +550,13 @@ class W_Socket(W_Root): try: if cmd == _c.SIO_RCVALL: option_ptr = rffi.cast(rffi.INTP, value_ptr) - option_ptr[0] = space.int_w(w_option) + option_ptr[0] = rffi.cast(rffi.INT, space.int_w(w_option)) elif cmd == _c.SIO_KEEPALIVE_VALS: w_onoff, w_time, w_interval = space.unpackiterable(w_option, 3) option_ptr = rffi.cast(lltype.Ptr(_c.tcp_keepalive), value_ptr) - option_ptr.c_onoff = space.uint_w(w_onoff) - option_ptr.c_keepalivetime = space.uint_w(w_time) - option_ptr.c_keepaliveinterval = space.uint_w(w_interval) + option_ptr.c_onoff = rffi.cast(rffi.UINT, space.uint_w(w_onoff)) + option_ptr.c_keepalivetime = rffi.cast(rffi.UINT, space.uint_w(w_time)) + option_ptr.c_keepaliveinterval = rffi.cast(rffi.UINT, space.uint_w(w_interval)) res = _c.WSAIoctl( self.sock.fd, cmd, value_ptr, value_size, diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py index 1bea4eb170..ec83da7f88 100644 --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -746,11 +746,7 @@ class W_SRE_Scanner(W_Root): match = W_SRE_Match(self.srepat, ctx) return match else: - # obscure corner case - if ctx.match_start == ctx.end: - self.ctx = None - else: - ctx.match_start = ctx.next_indirect(ctx.match_start) + self.ctx = None return None W_SRE_Scanner.typedef = TypeDef( diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py index fc7f972495..24885d36ac 100644 --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -119,6 +119,7 @@ class AppTestSrePattern: assert ["a", "u"] == re.findall("b(.)", "abalbus") assert [("a", "l"), ("u", "s")] == re.findall("b(.)(.)", "abalbus") assert [("a", ""), ("s", "s")] == re.findall("b(a|(s))", "babs") + assert ['', '', '', ''] == re.findall("X??", "1X4") # changes in 3.7 def test_findall_unicode(self): import re @@ -320,6 +321,10 @@ class AppTestSreMatch: import re assert re.sub('=\w{2}', 'x', '=CA') == 'x' + def test_sub_emptymatch(self): + import re + assert re.sub(r"b*", "*", "abc") == "*a*c*" # changes in 3.7 + def test_match_array(self): import re, array a = array.array('c', 'hello') @@ -403,9 +408,8 @@ class AppTestSreScanner: assert "a" == p.match().group(0) assert "a" == p.match().group(0) assert None == p.match() - assert "a" == p.match().group(0) - assert "a" == p.match().group(0) - assert None == p.match() + # the rest has been changed somewhere between Python 2.6.9 + # and Python 2.7.18. PyPy now follows the 2.7.18 behavior assert None == p.match() assert None == p.match() @@ -424,6 +428,13 @@ class AppTestSreScanner: assert ("bla", "") == (p.search().group(0), p.search().group(0)) assert None == p.search() + def test_scanner_empty_match(self): + import re, sys + p = re.compile("a??").scanner("bac") + assert ("", "", "", "") == (p.search().group(0), p.search().group(0), + p.search().group(0), p.search().group(0)) + assert None == p.search() + class AppTestGetlower: spaceconfig = dict(usemodules=('_locale',)) diff --git a/pypy/module/_warnings/test/apptest_warnings.py b/pypy/module/_warnings/test/apptest_warnings.py new file mode 100644 index 0000000000..2c5ce808ac --- /dev/null +++ b/pypy/module/_warnings/test/apptest_warnings.py @@ -0,0 +1,123 @@ +# spaceconfig = {"usemodules": ["_warnings"]} +import pytest + +import warnings +import _warnings + +import sys + +def test_defaults(): + assert _warnings.once_registry == {} + assert _warnings.default_action == 'default' + assert "PendingDeprecationWarning" in str(_warnings.filters) + +def test_warn(): + _warnings.warn("some message", DeprecationWarning) + _warnings.warn("some message", Warning) + _warnings.warn(("some message",1), Warning) + +def test_lineno(): + with warnings.catch_warnings(record=True) as w: + _warnings.warn("some message", Warning) + lineno = sys._getframe().f_lineno - 1 # the line above + assert w[-1].lineno == lineno + +def test_warn_explicit(): + _warnings.warn_explicit("some message", DeprecationWarning, + "<string>", 1, module_globals=globals()) + _warnings.warn_explicit("some message", Warning, + "<string>", 1, module_globals=globals()) + +def test_default_action(): + warnings.defaultaction = 'ignore' + warnings.resetwarnings() + with warnings.catch_warnings(record=True) as w: + __warningregistry__ = {} + _warnings.warn_explicit("message", UserWarning, "<test>", 44, + registry={}) + assert len(w) == 0 + warnings.defaultaction = 'default' + +def test_show_source_line(): + import sys, StringIO + try: + from test.warning_tests import inner + except ImportError: + skip('no test, -A on cpython?') + # With showarning() missing, make sure that output is okay. + saved = warnings.showwarning + try: + del warnings.showwarning + + stderr = sys.stderr + try: + sys.stderr = StringIO.StringIO() + inner('test message') + result = sys.stderr.getvalue() + finally: + sys.stderr = stderr + + assert result.count('\n') == 2 + assert ' warnings.warn(message, ' in result + finally: + warnings.showwarning = saved + + +def test_filename_none(): + globals()['__file__'] = 'test.pyc' + _warnings.warn('test', UserWarning) + globals()['__file__'] = None + _warnings.warn('test', UserWarning) + + +def test_warn_unicode(): + if '__pypy__' not in sys.builtin_module_names: + # see bc4acc4caa28 + pytest.skip("this checks that non-ascii warnings are not silently " + "swallowed, like they are with CPython 2.7 (buggily?)") + old = sys.stderr, warnings.showwarning + try: + class Grab: + def write(self, u): + self.data.append(u) + sys.stderr = Grab() + sys.stderr.data = data = [] + if sys.version_info > (3, 0, 0): + # Copy from lib-python/3/warnings.py + def orig_showwarning(message, category, filename, lineno, file=None, line=None): + msg = warnings.WarningMessage(message, category, filename, lineno, file, line) + warnings._showwarnmsg_impl(msg) + warnings.showwarning = orig_showwarning + _unicode = str + else: + warnings.showwarning = warnings._show_warning + _unicode = unicode + # ^^^ disables any catch_warnings() issued by the test runner + _warnings.warn_explicit("9238exbexn8", Warning, + "<string>", 1421, module_globals=globals()) + assert data # the warning was not swallowed + assert isinstance(''.join(data), str) + _warnings.warn_explicit(u"\u1234\u5678", UserWarning, + "<str2>", 831, module_globals=globals()) + assert isinstance(''.join(data), _unicode) + assert ''.join(data).endswith( + u'<str2>:831: UserWarning: \u1234\u5678\n') + finally: + sys.stderr, warnings.showwarning = old + + +def test_issue31285(): + def get_bad_loader(splitlines_ret_val): + class BadLoader: + def get_source(self, fullname): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + return BadSource('spam') + return BadLoader() + # does not raise: + _warnings.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals={'__loader__': get_bad_loader(42), + '__name__': 'foobar'}) + diff --git a/pypy/module/_warnings/test/test_warnings.py b/pypy/module/_warnings/test/test_warnings.py deleted file mode 100644 index c063d2adc0..0000000000 --- a/pypy/module/_warnings/test/test_warnings.py +++ /dev/null @@ -1,108 +0,0 @@ -class AppTestWarnings: - spaceconfig = dict(usemodules=('_warnings',)) - - def test_defaults(self): - import _warnings - assert _warnings.once_registry == {} - assert _warnings.default_action == 'default' - assert "PendingDeprecationWarning" in str(_warnings.filters) - - def test_warn(self): - import _warnings - _warnings.warn("some message", DeprecationWarning) - _warnings.warn("some message", Warning) - _warnings.warn(("some message",1), Warning) - - def test_lineno(self): - import warnings, _warnings, sys - with warnings.catch_warnings(record=True) as w: - _warnings.warn("some message", Warning) - lineno = sys._getframe().f_lineno - 1 # the line above - assert w[-1].lineno == lineno - - def test_warn_explicit(self): - import _warnings - _warnings.warn_explicit("some message", DeprecationWarning, - "<string>", 1, module_globals=globals()) - _warnings.warn_explicit("some message", Warning, - "<string>", 1, module_globals=globals()) - - def test_default_action(self): - import warnings, _warnings - warnings.defaultaction = 'ignore' - warnings.resetwarnings() - with warnings.catch_warnings(record=True) as w: - __warningregistry__ = {} - _warnings.warn_explicit("message", UserWarning, "<test>", 44, - registry={}) - assert len(w) == 0 - warnings.defaultaction = 'default' - - def test_show_source_line(self): - import warnings - import sys, StringIO - try: - from test.warning_tests import inner - except ImportError: - skip('no test, -A on cpython?') - # With showarning() missing, make sure that output is okay. - saved = warnings.showwarning - try: - del warnings.showwarning - - stderr = sys.stderr - try: - sys.stderr = StringIO.StringIO() - inner('test message') - result = sys.stderr.getvalue() - finally: - sys.stderr = stderr - - assert result.count('\n') == 2 - assert ' warnings.warn(message, ' in result - finally: - warnings.showwarning = saved - - def test_filename_none(self): - import _warnings - globals()['__file__'] = 'test.pyc' - _warnings.warn('test', UserWarning) - globals()['__file__'] = None - _warnings.warn('test', UserWarning) - - def test_warn_unicode(self): - import _warnings, sys - old = sys.stderr - try: - class Grab: - def write(self, u): - self.data.append(u) - sys.stderr = Grab() - sys.stderr.data = data = [] - _warnings.warn_explicit("9238exbexn8", Warning, - "<string>", 1421, module_globals=globals()) - assert isinstance(''.join(data), str) - _warnings.warn_explicit(u"\u1234\u5678", UserWarning, - "<str2>", 831, module_globals=globals()) - assert isinstance(''.join(data), unicode) - assert ''.join(data).endswith( - u'<str2>:831: UserWarning: \u1234\u5678\n') - finally: - sys.stderr = old - - def test_issue31285(self): - import _warnings - def get_bad_loader(splitlines_ret_val): - class BadLoader: - def get_source(self, fullname): - class BadSource(str): - def splitlines(self): - return splitlines_ret_val - return BadSource('spam') - return BadLoader() - # does not raise: - _warnings.warn_explicit( - 'eggs', UserWarning, 'bar', 1, - module_globals={'__loader__': get_bad_loader(42), - '__name__': 'foobar'}) - diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py index 448b443713..dc29cd844c 100644 --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -6,6 +6,61 @@ from pypy.interpreter.error import OperationError, oefmt, wrap_windowserror from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import rwinreg, rwin32 from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +# wrappers needed to call the reflection functions loaded at runtime +# using WINAPI convention +eci = ExternalCompilationInfo( + includes=['windows.h'], + post_include_bits=[ + "RPY_EXTERN LONG\n" + "pypy_RegChangeReflectionKey(FARPROC address, HKEY key);\n" + "RPY_EXTERN LONG\n" + "pypy_RegQueryReflectionKey(FARPROC address, HKEY key, LPBOOL isDisabled);\n" + "RPY_EXTERN LONG\n" + "pypy_RegDeleteKeyExA(FARPROC address, HKEY key, LPCSTR subkey,\n" + " REGSAM sam, DWORD reserved);\n" + ], + separate_module_sources=[''' + LONG + pypy_RegChangeReflectionKey(FARPROC address, HKEY key) { + LONG (WINAPI *func)(HKEY); + *(FARPROC*)&func = address; + return func(key); + } + + LONG + pypy_RegQueryReflectionKey(FARPROC address, HKEY key, LPBOOL isDisabled) { + LONG (WINAPI *func)(HKEY, LPBOOL); + *(FARPROC*)&func = address; + return func(key, isDisabled); + } + + LONG + pypy_RegDeleteKeyExA(FARPROC address, HKEY key, LPCSTR subkey, + REGSAM sam, DWORD reserved) { + LONG (WINAPI *func)(HKEY, LPCSTR, REGSAM, DWORD); + *(FARPROC*)&func = address; + return func(key, subkey, sam, reserved); + } + '''], +) +pypy_RegChangeReflectionKey = rffi.llexternal( + 'pypy_RegChangeReflectionKey', + [rffi.VOIDP, rwinreg.HKEY], + rffi.LONG, compilation_info=eci) + +pypy_RegQueryReflectionKey = rffi.llexternal( + 'pypy_RegQueryReflectionKey', + [rffi.VOIDP, rwinreg.HKEY, rwin32.LPBOOL], + rffi.LONG, compilation_info=eci) + +pypy_RegDeleteKeyExA = rffi.llexternal( + 'pypy_RegDeleteKeyExA', + [rffi.VOIDP, rwinreg.HKEY, rffi.CCHARP, rwinreg.REGSAM, rwin32.DWORD], + rffi.LONG, compilation_info=eci) + def raiseWindowsError(space, errcode, context): message = rwin32.FormatError(errcode) @@ -57,8 +112,7 @@ handle is already detached, this will return zero. After calling this function, the handle is effectively invalidated, but the handle is not closed. You would call this function when you need the underlying win32 handle to exist beyond the lifetime of the -handle object. -On 64 bit windows, the result of this function is a long integer""" +handle object.""" key = self.as_int() self.hkey = rwin32.NULL_HANDLE return space.newint(key) @@ -253,13 +307,15 @@ But the underlying API call doesn't return the type, Lame Lame Lame, DONT USE TH if ret == rwinreg.ERROR_MORE_DATA: # Resize and retry bufSize *= 2 - bufsize_p[0] = bufSize + bufsize_p[0] = rffi.cast(rwin32.LONG, bufSize) continue if ret != 0: raiseWindowsError(space, ret, 'RegQueryValue') - length = intmask(bufsize_p[0] - 1) - return space.newtext(rffi.charp2strn(buf, length)) + length = intmask(bufsize_p[0]) + if length == 0: + return space.w_None + return space.newtext(rffi.charp2strn(buf, length - 1)) def convert_to_regdata(space, w_value, typ): buf = None @@ -274,7 +330,7 @@ def convert_to_regdata(space, w_value, typ): value = space.c_uint_w(w_value) buflen = rffi.sizeof(rwin32.DWORD) buf1 = lltype.malloc(rffi.CArray(rwin32.DWORD), 1, flavor='raw') - buf1[0] = value + buf1[0] = rffi.cast(rffi.UINT, value) buf = rffi.cast(rffi.CCHARP, buf1) elif typ == rwinreg.REG_SZ or typ == rwinreg.REG_EXPAND_SZ: @@ -329,8 +385,7 @@ def convert_to_regdata(space, w_value, typ): else: # REG_BINARY and ALL unknown data types. if space.is_w(w_value, space.w_None): buflen = 0 - buf = lltype.malloc(rffi.CCHARP.TO, 1, flavor='raw') - buf[0] = '\0' + buf = lltype.nullptr(rffi.CCHARP.TO) else: try: value = w_value.readbuf_w(space) @@ -386,7 +441,10 @@ def convert_from_regdata(space, buf, buflen, typ): return space.newlist(l) else: # REG_BINARY and all other types - return space.newbytes(rffi.charpsize2str(buf, buflen)) + if buflen == 0: + return space.w_None + else: + return space.newbytes(rffi.charpsize2str(buf, buflen)) @unwrap_spec(value_name="text", typ=int) def SetValueEx(space, w_hkey, value_name, w_reserved, typ, w_value): @@ -425,7 +483,8 @@ the configuration registry. This helps the registry perform efficiently.""" try: ret = rwinreg.RegSetValueExA(hkey, value_name, 0, typ, buf, buflen) finally: - lltype.free(buf, flavor='raw') + if buf != lltype.nullptr(rffi.CCHARP.TO): + lltype.free(buf, flavor='raw') if ret != 0: raiseWindowsError(space, ret, 'RegSetValueEx') @@ -463,10 +522,11 @@ value_name is a string indicating the value to query""" if ret != 0: raiseWindowsError(space, ret, 'RegQueryValueEx') length = intmask(retDataSize[0]) + ret_type = intmask(retType[0]) return space.newtuple([ convert_from_regdata(space, databuf, - length, retType[0]), - space.newint(intmask(retType[0])), + length, ret_type), + space.newint(intmask(ret_type)), ]) @unwrap_spec(subkey="text") @@ -489,7 +549,7 @@ If the function fails, an exception is raised.""" raiseWindowsError(space, ret, 'CreateKey') return W_HKEY(space, rethkey[0]) -@unwrap_spec(subkey="text", res=int, sam=rffi.r_uint) +@unwrap_spec(subkey="text", res=int, sam=r_uint) def CreateKeyEx(space, w_hkey, subkey, res=0, sam=rwinreg.KEY_WRITE): """key = CreateKey(key, sub_key) - Creates or opens the specified key. @@ -539,7 +599,7 @@ value is a string that identifies the value to remove.""" if ret != 0: raiseWindowsError(space, ret, 'RegDeleteValue') -@unwrap_spec(subkey="text", res=int, sam=rffi.r_uint) +@unwrap_spec(subkey="text", res=int, sam=r_uint) def OpenKey(space, w_hkey, subkey, res=0, sam=rwinreg.KEY_READ): """key = OpenKey(key, sub_key, res = 0, sam = KEY_READ) - Opens the specified key. @@ -586,10 +646,10 @@ data_type is an integer that identifies the type of the value data.""" if ret != 0: raiseWindowsError(space, ret, 'RegQueryInfoKey') # include null terminators - retValueSize[0] += 1 - retDataSize[0] += 1 - bufDataSize = intmask(retDataSize[0]) - bufValueSize = intmask(retValueSize[0]) + bufDataSize = intmask(retDataSize[0]) + 1 + bufValueSize = intmask(retValueSize[0]) + 1 + retValueSize[0] = rffi.cast(rwin32.DWORD, bufValueSize) + retDataSize[0] = rffi.cast(rwin32.DWORD, bufDataSize) with lltype.scoped_alloc(rffi.CCHARP.TO, intmask(retValueSize[0])) as valuebuf: @@ -614,11 +674,12 @@ data_type is an integer that identifies the type of the value data.""" raiseWindowsError(space, ret, 'RegEnumValue') length = intmask(retDataSize[0]) + ret_type = intmask(retType[0]) return space.newtuple([ space.newtext(rffi.charp2str(valuebuf)), convert_from_regdata(space, databuf, - length, retType[0]), - space.newint(intmask(retType[0])), + length, ret_type), + space.newint(ret_type), ]) @unwrap_spec(index=int) @@ -642,7 +703,7 @@ raised, indicating no more values are available.""" # retrieve such a key name. with lltype.scoped_alloc(rffi.CCHARP.TO, 257) as buf: with lltype.scoped_alloc(rwin32.LPDWORD.TO, 1) as retValueSize: - retValueSize[0] = r_uint(257) # includes NULL terminator + retValueSize[0] = rffi.r_uint(257) # includes NULL terminator ret = rwinreg.RegEnumKeyExA(hkey, index, buf, retValueSize, null_dword, None, null_dword, lltype.nullptr(rwin32.PFILETIME.TO)) @@ -706,30 +767,84 @@ def ExpandEnvironmentStrings(space, w_source): except WindowsError as e: raise wrap_windowserror(space, e) + +class ReflectionFunction(object): + def __init__(self, name, stdcall_wrapper): + self.name = name + self.handle = lltype.nullptr(rffi.VOIDP.TO) + self.wrapper = stdcall_wrapper + + def check(self): + if self.handle != lltype.nullptr(rffi.VOIDP.TO): + return True + from rpython.rlib.rdynload import GetModuleHandle, dlsym + lib = GetModuleHandle("advapi32.dll") + try: + handle = dlsym(lib, self.name) + except KeyError: + return False + self.handle = handle + return True + + def call(self, *args): + assert self.handle != lltype.nullptr(rffi.VOIDP.TO) + return self.wrapper(self.handle, *args) + + +_RegDisableReflectionKey = ReflectionFunction( + "RegDisableReflectionKey", pypy_RegChangeReflectionKey) +_RegEnableReflectionKey = ReflectionFunction( + "RegEnableReflectionKey", pypy_RegChangeReflectionKey) +_RegQueryReflectionKey = ReflectionFunction( + "RegQueryReflectionKey", pypy_RegQueryReflectionKey) +_RegDeleteKeyExA = ReflectionFunction("RegDeleteKeyExA", pypy_RegDeleteKeyExA) + + def DisableReflectionKey(space, w_key): """Disables registry reflection for 32-bit processes running on a 64-bit Operating System. Will generally raise NotImplemented if executed on a 32-bit Operating System. If the key is not on the reflection list, the function succeeds but has no effect. Disabling reflection for a key does not affect reflection of any subkeys.""" - raise oefmt(space.w_NotImplementedError, - "not implemented on this platform") + if not _RegDisableReflectionKey.check(): + raise oefmt(space.w_NotImplementedError, + "not implemented on this platform") + else: + hkey = hkey_w(w_key, space) + ret = _RegDisableReflectionKey.call(hkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegDisableReflectionKey') def EnableReflectionKey(space, w_key): """Restores registry reflection for the specified disabled key. Will generally raise NotImplemented if executed on a 32-bit Operating System. Restoring reflection for a key does not affect reflection of any subkeys.""" - raise oefmt(space.w_NotImplementedError, - "not implemented on this platform") + if not _RegEnableReflectionKey.check(): + raise oefmt(space.w_NotImplementedError, + "not implemented on this platform") + else: + hkey = hkey_w(w_key, space) + ret = _RegEnableReflectionKey.call(hkey) + if ret != 0: + raiseWindowsError(space, ret, 'RegEnableReflectionKey') def QueryReflectionKey(space, w_key): """bool = QueryReflectionKey(hkey) - Determines the reflection state for the specified key. Will generally raise NotImplemented if executed on a 32-bit Operating System.""" - raise oefmt(space.w_NotImplementedError, - "not implemented on this platform") + if not _RegQueryReflectionKey.check(): + raise oefmt(space.w_NotImplementedError, + "not implemented on this platform") + else: + hkey = hkey_w(w_key, space) + with lltype.scoped_alloc(rwin32.LPBOOL.TO, 1) as isDisabled: + ret = _RegQueryReflectionKey.call(hkey, isDisabled) + if ret != 0: + raiseWindowsError(space, ret, 'RegQueryReflectionKey') + return space.newbool(intmask(isDisabled[0]) != 0) -@unwrap_spec(subkey="text") -def DeleteKeyEx(space, w_key, subkey): + +@unwrap_spec(sub_key="text", access=r_uint, reserved=int) +def DeleteKeyEx(space, w_key, sub_key, access=rwinreg.KEY_WOW64_64KEY, reserved=0): """DeleteKeyEx(key, sub_key, sam, res) - Deletes the specified key. key is an already open key, or any one of the predefined HKEY_* constants. @@ -743,5 +858,11 @@ def DeleteKeyEx(space, w_key, subkey): If the method succeeds, the entire key, including all of its values, is removed. If the method fails, a WindowsError exception is raised. On unsupported Windows versions, NotImplementedError is raised.""" - raise oefmt(space.w_NotImplementedError, - "not implemented on this platform") + if not _RegDeleteKeyExA.check(): + raise oefmt(space.w_NotImplementedError, + "not implemented on this platform") + else: + hkey = hkey_w(w_key, space) + ret = _RegDeleteKeyExA.call(hkey, sub_key, access, reserved) + if ret != 0: + raiseWindowsError(space, ret, 'RegDeleteKeyEx') diff --git a/pypy/module/_winreg/moduledef.py b/pypy/module/_winreg/moduledef.py index d865ca691e..b20bd15d62 100644 --- a/pypy/module/_winreg/moduledef.py +++ b/pypy/module/_winreg/moduledef.py @@ -72,3 +72,8 @@ to see what constants are used, and where.""" for name, value in constants.iteritems(): interpleveldefs[name] = "space.wrap(%s)" % (value,) + + import pypy.module.sys.version + if pypy.module.sys.version.CPYTHON_VERSION < (3, 6): + del interpleveldefs["REG_QWORD"] + del interpleveldefs["REG_QWORD_LITTLE_ENDIAN"] diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py index ae2d8ee227..1d25cbb900 100644 --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -31,6 +31,7 @@ class AppTestFfi: def setup_class(cls): import _winreg + from platform import machine space = cls.space cls.root_key = _winreg.HKEY_CURRENT_USER cls.test_key_name = "SOFTWARE\\Pypy Registry Test Key - Delete Me" @@ -38,6 +39,7 @@ class AppTestFfi: cls.w_test_key_name = space.wrap(cls.test_key_name) cls.w_canSaveKey = space.wrap(canSaveKey) cls.w_tmpfilename = space.wrap(str(udir.join('winreg-temp'))) + cls.w_win64_machine = space.wrap(machine() == "AMD64") test_data = [ ("Int Value", 0xFEDCBA98, _winreg.REG_DWORD), @@ -45,6 +47,7 @@ class AppTestFfi: ("Unicode Value", u"A unicode Value", _winreg.REG_SZ), ("Str Expand", "The path is %path%", _winreg.REG_EXPAND_SZ), ("Multi Str", ["Several", "string", u"values"], _winreg.REG_MULTI_SZ), + ("Raw None", None, _winreg.REG_BINARY), ("Raw data", "binary"+chr(0)+"data", _winreg.REG_BINARY), ] cls.w_test_data = space.wrap(test_data) @@ -175,14 +178,19 @@ class AppTestFfi: def test_delete(self): # must be run after test_SetValueEx - from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey + from _winreg import OpenKey, KEY_ALL_ACCESS, DeleteValue, DeleteKey, DeleteKeyEx key = OpenKey(self.root_key, self.test_key_name, 0, KEY_ALL_ACCESS) sub_key = OpenKey(key, "sub_key", 0, KEY_ALL_ACCESS) for name, value, type in self.test_data: DeleteValue(sub_key, name) - DeleteKey(key, "sub_key") + if self.win64_machine: + DeleteKeyEx(key, "sub_key", KEY_ALL_ACCESS, 0) + else: + DeleteKey(key, "sub_key") + + raises(OSError, OpenKey, key, "sub_key") def test_connect(self): from _winreg import ConnectRegistry, HKEY_LOCAL_MACHINE @@ -255,3 +263,18 @@ class AppTestFfi: raises(NotImplementedError, DeleteKeyEx, self.root_key, self.test_key_name) + def test_reflection(self): + import sys + from _winreg import DisableReflectionKey, EnableReflectionKey, \ + QueryReflectionKey, OpenKey, HKEY_LOCAL_MACHINE + # Adapted from lib-python test + if not self.win64_machine: + skip("Requires 64-bit host") + # Test that we can call the query, enable, and disable functions + # on a key which isn't on the reflection list with no consequences. + with OpenKey(HKEY_LOCAL_MACHINE, "Software") as key: + # HKLM\Software is redirected but not reflected in all OSes + assert QueryReflectionKey(key) + assert EnableReflectionKey(key) is None + assert DisableReflectionKey(key) is None + assert QueryReflectionKey(key) diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py index a628e28402..8b3a4a80f9 100644 --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -816,14 +816,23 @@ class TypeCode(object): # hint for the annotator: track individual constant instances return True -if rffi.sizeof(rffi.UINT) == rffi.sizeof(rffi.ULONG): - # 32 bits: UINT can't safely overflow into a C long (rpython int) +if rffi.sizeof(rffi.UINT) == rffi.sizeof(lltype.Unsigned): + # 32 bits: UINT can't safely overflow into a lltype.Unsigned (rpython int) # via int_w, handle it like ULONG below _UINTTypeCode = \ TypeCode(rffi.UINT, 'bigint_w') else: _UINTTypeCode = \ TypeCode(rffi.UINT, 'int_w', True) +if rffi.sizeof(rffi.ULONG) == rffi.sizeof(lltype.Unsigned): + # Overflow handled by rbigint.touint() which + # corresponds to lltype.Unsigned + _ULONGTypeCode = \ + TypeCode(rffi.ULONG, 'bigint_w', errorname="integer") +else: + # 64 bit Windows special case: ULONG is same as UINT + _ULONGTypeCode = \ + TypeCode(rffi.ULONG, 'int_w', True, errorname="integer") types = { 'c': TypeCode(lltype.Char, 'bytes_w', method=''), 'u': TypeCode(lltype.UniChar, 'utf8_len_w', method=''), @@ -834,10 +843,7 @@ types = { 'i': TypeCode(rffi.INT, 'int_w', True, True), 'I': _UINTTypeCode, 'l': TypeCode(rffi.LONG, 'int_w', True, True), - 'L': TypeCode(rffi.ULONG, 'bigint_w', # Overflow handled by - errorname="integer"), # rbigint.touint() which - # corresponds to the - # C-type unsigned long + 'L': _ULONGTypeCode, 'f': TypeCode(lltype.SingleFloat, 'float_w', method='__float__'), 'd': TypeCode(lltype.Float, 'float_w', method='__float__'), } diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py index 7a2ec8c92c..cc13021efd 100644 --- a/pypy/module/array/test/test_array.py +++ b/pypy/module/array/test/test_array.py @@ -89,9 +89,10 @@ class AppTestArray(object): assert a[i] == v assert type(a[i]) is pt or ( # A special case: we return ints in Array('I') on 64-bits, + # and in Array('L') on 64-bit Windows, # whereas CPython returns longs. The difference is # probably acceptable. - tc == 'I' and + (tc == 'I' or tc == 'L' and sys.platform == 'win32') and sys.maxint > 2147483647 and type(a[i]) is int) for v in ok: a[1] = v diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py index d52c232925..60dc9f30b3 100644 --- a/pypy/module/bz2/interp_bz2.py +++ b/pypy/module/bz2/interp_bz2.py @@ -34,7 +34,7 @@ class CConfig: CHECK_LIBRARY = platform.Has('dump("x", (long)&BZ2_bzCompress)') off_t = platform.SimpleType("off_t", rffi.LONGLONG) - size_t = platform.SimpleType("size_t", rffi.ULONG) + size_t = platform.SimpleType("size_t", rffi.UNSIGNED) BUFSIZ = platform.ConstantInteger("BUFSIZ") _alloc_type = lltype.FuncType([rffi.VOIDP, rffi.INT, rffi.INT], rffi.VOIDP) _free_type = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], lltype.Void) @@ -106,7 +106,7 @@ else: BIGCHUNK = 512 * 1024 if BZ_CONFIG_ERROR: - if rffi.sizeof(rffi.LONG) >= 8: + if rffi.sizeof(rffi.SIGNED) >= 8: def _bzs_total_out(bzs): return (rffi.getintfield(bzs, 'c_total_out_hi32') << 32) + \ rffi.getintfield(bzs, 'c_total_out_lo32') diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py index 6d540139de..84ec9ed150 100644 --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -42,7 +42,7 @@ from rpython.rlib import rthread from rpython.rlib.debug import fatalerror_notb from rpython.rlib import rstackovf from pypy.objspace.std.typeobject import W_TypeObject, find_best_base -from pypy.module.cpyext.cparser import CTypeSpace +from rpython.tool.cparser import CTypeSpace DEBUG_WRAPPER = True @@ -131,7 +131,7 @@ udir.join('pypy_macros.h').write("/* Will be filled later */\n") constant_names = """ Py_TPFLAGS_READY Py_TPFLAGS_READYING Py_TPFLAGS_HAVE_GETCHARBUFFER -METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE Py_MAX_FMT +METH_COEXIST METH_STATIC METH_CLASS Py_TPFLAGS_BASETYPE METH_NOARGS METH_VARARGS METH_KEYWORDS METH_O Py_TPFLAGS_HAVE_INPLACEOPS Py_TPFLAGS_HEAPTYPE Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_HAVE_NEWBUFFER Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS @@ -724,11 +724,13 @@ class CpyextTypeSpace(CTypeSpace): CPYEXT_BASE_HEADERS = ['sys/types.h', 'stdarg.h', 'stdio.h', 'stddef.h'] cts = CpyextTypeSpace(headers=CPYEXT_BASE_HEADERS) -cts.parse_header(parse_dir / 'cpyext_object.h') +cts.parse_header(parse_dir / 'cpyext_object.h', configure=False) +cts.parse_header(parse_dir / 'cpyext_descrobject.h', configure=False) +cts.configure_types() Py_ssize_t = cts.gettype('Py_ssize_t') Py_ssize_tP = cts.gettype('Py_ssize_t *') -size_t = rffi.ULONG +size_t = lltype.Unsigned ADDR = lltype.Signed # Note: as a special case, "PyObject" is the pointer type in RPython, @@ -1369,21 +1371,16 @@ def mangle_name(prefix, name): def write_header(header_name, decls): lines = [ + '#include "cpyext_object.h"', ''' #ifdef _WIN64 -/* this check is for sanity, but also because the 'temporary fix' - below seems to become permanent and would cause unexpected - nonsense on Win64---but note that it's not the only reason for - why Win64 is not supported! If you want to help, see - http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation - */ -# error "PyPy does not support 64-bit on Windows. Use Win32" +#define Signed Py_ssize_t /* xxx temporary fix */ +#define Unsigned unsigned long long /* xxx temporary fix */ +#else +#define Signed Py_ssize_t /* xxx temporary fix */ +#define Unsigned unsigned long /* xxx temporary fix */ #endif -''', - '#include "cpyext_object.h"', - '#define Signed Py_ssize_t /* xxx temporary fix */', - '#define Unsigned unsigned long /* xxx temporary fix */', - '',] + decls + [ + '''] + decls + [ '', '#undef Signed /* xxx temporary fix */', '#undef Unsigned /* xxx temporary fix */', diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py index aef1e27a8a..a4ae8ccfc2 100644 --- a/pypy/module/cpyext/bytesobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -143,7 +143,7 @@ def _PyString_AsString(space, ref): elif not PyString_Check(space, ref): # otherwise, use the alternate way from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) - if PyUnicode_Check(space, ref): + if PyUnicode_Check(ref): ref = _PyUnicode_AsDefaultEncodedString(space, ref, lltype.nullptr(rffi.CCHARP.TO)) else: raise oefmt(space.w_TypeError, @@ -167,7 +167,7 @@ def PyString_AsStringAndSize(space, ref, data, length): if not PyString_Check(space, ref): from pypy.module.cpyext.unicodeobject import ( PyUnicode_Check, _PyUnicode_AsDefaultEncodedString) - if PyUnicode_Check(space, ref): + if PyUnicode_Check(ref): ref = _PyUnicode_AsDefaultEncodedString(space, ref, lltype.nullptr(rffi.CCHARP.TO)) else: raise oefmt(space.w_TypeError, diff --git a/pypy/module/cpyext/cmodel.py b/pypy/module/cpyext/cmodel.py deleted file mode 100644 index bea523da15..0000000000 --- a/pypy/module/cpyext/cmodel.py +++ /dev/null @@ -1,607 +0,0 @@ -import types -import weakref - -from .error import CDefError, VerificationError, VerificationMissing - -# type qualifiers -Q_CONST = 0x01 -Q_RESTRICT = 0x02 -Q_VOLATILE = 0x04 - -def qualify(quals, replace_with): - if quals & Q_CONST: - replace_with = ' const ' + replace_with.lstrip() - if quals & Q_VOLATILE: - replace_with = ' volatile ' + replace_with.lstrip() - if quals & Q_RESTRICT: - # It seems that __restrict is supported by gcc and msvc. - # If you hit some different compiler, add a #define in - # _cffi_include.h for it (and in its copies, documented there) - replace_with = ' __restrict ' + replace_with.lstrip() - return replace_with - - -class BaseTypeByIdentity(object): - is_array_type = False - is_raw_function = False - - def get_c_name(self, replace_with='', context='a C file', quals=0): - result = self.c_name_with_marker - assert result.count('&') == 1 - # some logic duplication with ffi.getctype()... :-( - replace_with = replace_with.strip() - if replace_with: - if replace_with.startswith('*') and '&[' in result: - replace_with = '(%s)' % replace_with - elif not replace_with[0] in '[(': - replace_with = ' ' + replace_with - replace_with = qualify(quals, replace_with) - result = result.replace('&', replace_with) - if '$' in result: - raise VerificationError( - "cannot generate '%s' in %s: unknown type name" - % (self._get_c_name(), context)) - return result - - def _get_c_name(self): - return self.c_name_with_marker.replace('&', '') - - def has_c_name(self): - return '$' not in self._get_c_name() - - def is_integer_type(self): - return False - - def get_cached_btype(self, ffi, finishlist, can_delay=False): - try: - BType = ffi._cached_btypes[self] - except KeyError: - BType = self.build_backend_type(ffi, finishlist) - BType2 = ffi._cached_btypes.setdefault(self, BType) - assert BType2 is BType - return BType - - def __repr__(self): - return '<%s>' % (self._get_c_name(),) - - def _get_items(self): - return [(name, getattr(self, name)) for name in self._attrs_] - - -class BaseType(BaseTypeByIdentity): - - def __eq__(self, other): - return (self.__class__ == other.__class__ and - self._get_items() == other._get_items()) - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash((self.__class__, tuple(self._get_items()))) - - -class VoidType(BaseType): - _attrs_ = () - - def __init__(self): - self.c_name_with_marker = 'void&' - - def build_backend_type(self, ffi, finishlist): - return global_cache(self, ffi, 'new_void_type') - -void_type = VoidType() - - -class BasePrimitiveType(BaseType): - pass - - -class PrimitiveType(BasePrimitiveType): - _attrs_ = ('name',) - - ALL_PRIMITIVE_TYPES = { - 'char': 'c', - 'short': 'i', - 'int': 'i', - 'long': 'i', - 'long long': 'i', - 'signed char': 'i', - 'unsigned char': 'i', - 'unsigned short': 'i', - 'unsigned int': 'i', - 'unsigned long': 'i', - 'unsigned long long': 'i', - 'float': 'f', - 'double': 'f', - 'long double': 'f', - '_Bool': 'i', - # the following types are not primitive in the C sense - 'wchar_t': 'c', - 'int8_t': 'i', - 'uint8_t': 'i', - 'int16_t': 'i', - 'uint16_t': 'i', - 'int32_t': 'i', - 'uint32_t': 'i', - 'int64_t': 'i', - 'uint64_t': 'i', - 'int_least8_t': 'i', - 'uint_least8_t': 'i', - 'int_least16_t': 'i', - 'uint_least16_t': 'i', - 'int_least32_t': 'i', - 'uint_least32_t': 'i', - 'int_least64_t': 'i', - 'uint_least64_t': 'i', - 'int_fast8_t': 'i', - 'uint_fast8_t': 'i', - 'int_fast16_t': 'i', - 'uint_fast16_t': 'i', - 'int_fast32_t': 'i', - 'uint_fast32_t': 'i', - 'int_fast64_t': 'i', - 'uint_fast64_t': 'i', - 'intptr_t': 'i', - 'uintptr_t': 'i', - 'intmax_t': 'i', - 'uintmax_t': 'i', - 'ptrdiff_t': 'i', - 'size_t': 'i', - 'ssize_t': 'i', - } - - def __init__(self, name): - assert name in self.ALL_PRIMITIVE_TYPES - self.name = name - self.c_name_with_marker = name + '&' - - def is_char_type(self): - return self.ALL_PRIMITIVE_TYPES[self.name] == 'c' - def is_integer_type(self): - return self.ALL_PRIMITIVE_TYPES[self.name] == 'i' - def is_float_type(self): - return self.ALL_PRIMITIVE_TYPES[self.name] == 'f' - - def build_backend_type(self, ffi, finishlist): - return global_cache(self, ffi, 'new_primitive_type', self.name) - - -class UnknownIntegerType(BasePrimitiveType): - _attrs_ = ('name',) - - def __init__(self, name): - self.name = name - self.c_name_with_marker = name + '&' - - def is_integer_type(self): - return True - - def build_backend_type(self, ffi, finishlist): - raise NotImplementedError("integer type '%s' can only be used after " - "compilation" % self.name) - -class UnknownFloatType(BasePrimitiveType): - _attrs_ = ('name', ) - - def __init__(self, name): - self.name = name - self.c_name_with_marker = name + '&' - - def build_backend_type(self, ffi, finishlist): - raise NotImplementedError("float type '%s' can only be used after " - "compilation" % self.name) - -class DefinedType(BaseType): - _attrs_ = ('name', ) - - def __init__(self, name, realtype, quals): - self.name = name - self.realtype = realtype - self.quals = quals - self.c_name_with_marker = name + '&' - - -class BaseFunctionType(BaseType): - _attrs_ = ('args', 'result', 'ellipsis', 'abi') - - def __init__(self, args, result, ellipsis, abi=None): - self.args = args - self.result = result - self.ellipsis = ellipsis - self.abi = abi - # - reprargs = [arg._get_c_name() for arg in self.args] - if self.ellipsis: - reprargs.append('...') - reprargs = reprargs or ['void'] - replace_with = self._base_pattern % (', '.join(reprargs),) - if abi is not None: - replace_with = replace_with[:1] + abi + ' ' + replace_with[1:] - self.c_name_with_marker = ( - self.result.c_name_with_marker.replace('&', replace_with)) - - -class RawFunctionType(BaseFunctionType): - # Corresponds to a C type like 'int(int)', which is the C type of - # a function, but not a pointer-to-function. The backend has no - # notion of such a type; it's used temporarily by parsing. - _base_pattern = '(&)(%s)' - is_raw_function = True - - def build_backend_type(self, ffi, finishlist): - raise CDefError("cannot render the type %r: it is a function " - "type, not a pointer-to-function type" % (self,)) - - def as_function_pointer(self): - return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) - - -class FunctionPtrType(BaseFunctionType): - _base_pattern = '(*&)(%s)' - - def build_backend_type(self, ffi, finishlist): - result = self.result.get_cached_btype(ffi, finishlist) - args = [] - for tp in self.args: - args.append(tp.get_cached_btype(ffi, finishlist)) - abi_args = () - if self.abi == "__stdcall": - if not self.ellipsis: # __stdcall ignored for variadic funcs - try: - abi_args = (ffi._backend.FFI_STDCALL,) - except AttributeError: - pass - return global_cache(self, ffi, 'new_function_type', - tuple(args), result, self.ellipsis, *abi_args) - - def as_raw_function(self): - return RawFunctionType(self.args, self.result, self.ellipsis, self.abi) - - -class PointerType(BaseType): - _attrs_ = ('totype', 'quals') - - def __init__(self, totype, quals=0): - self.totype = totype - self.quals = quals - extra = qualify(quals, " *&") - if totype.is_array_type: - extra = "(%s)" % (extra.lstrip(),) - self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) - - def build_backend_type(self, ffi, finishlist): - BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) - return global_cache(self, ffi, 'new_pointer_type', BItem) - -voidp_type = PointerType(void_type) - -def ConstPointerType(totype): - return PointerType(totype, Q_CONST) - -const_voidp_type = ConstPointerType(void_type) - - -class NamedPointerType(PointerType): - _attrs_ = ('totype', 'name') - - def __init__(self, totype, name, quals=0): - PointerType.__init__(self, totype, quals) - self.name = name - self.c_name_with_marker = name + '&' - - -class ArrayType(BaseType): - _attrs_ = ('item', 'length') - is_array_type = True - - def __init__(self, item, length): - self.item = item - self.length = length - # - if length is None: - brackets = '&[]' - elif length == '...': - brackets = '&[/*...*/]' - else: - brackets = '&[%s]' % length - self.c_name_with_marker = ( - self.item.c_name_with_marker.replace('&', brackets)) - - def resolve_length(self, newlength): - return ArrayType(self.item, newlength) - - def build_backend_type(self, ffi, finishlist): - if self.length == '...': - raise CDefError("cannot render the type %r: unknown length" % - (self,)) - self.item.get_cached_btype(ffi, finishlist) # force the item BType - BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) - return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) - -char_array_type = ArrayType(PrimitiveType('char'), None) - - -class StructOrUnionOrEnum(BaseTypeByIdentity): - _attrs_ = ('name',) - forcename = None - - def build_c_name_with_marker(self): - name = self.forcename or '%s %s' % (self.kind, self.name) - self.c_name_with_marker = name + '&' - - def force_the_name(self, forcename): - self.forcename = forcename - self.build_c_name_with_marker() - - def get_official_name(self): - assert self.c_name_with_marker.endswith('&') - return self.c_name_with_marker[:-1] - - -class StructOrUnion(StructOrUnionOrEnum): - fixedlayout = None - completed = 0 - partial = False - packed = False - - def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): - self.name = name - self.fldnames = fldnames - self.fldtypes = fldtypes - self.fldbitsize = fldbitsize - self.fldquals = fldquals - self.build_c_name_with_marker() - - def has_anonymous_struct_fields(self): - if self.fldtypes is None: - return False - for name, type in zip(self.fldnames, self.fldtypes): - if name == '' and isinstance(type, StructOrUnion): - return True - return False - - def enumfields(self): - fldquals = self.fldquals - if fldquals is None: - fldquals = (0,) * len(self.fldnames) - for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, - self.fldbitsize, fldquals): - if name == '' and isinstance(type, StructOrUnion): - # nested anonymous struct/union - for result in type.enumfields(): - yield result - else: - yield (name, type, bitsize, quals) - - def force_flatten(self): - # force the struct or union to have a declaration that lists - # directly all fields returned by enumfields(), flattening - # nested anonymous structs/unions. - names = [] - types = [] - bitsizes = [] - fldquals = [] - for name, type, bitsize, quals in self.enumfields(): - names.append(name) - types.append(type) - bitsizes.append(bitsize) - fldquals.append(quals) - self.fldnames = tuple(names) - self.fldtypes = tuple(types) - self.fldbitsize = tuple(bitsizes) - self.fldquals = tuple(fldquals) - - def get_cached_btype(self, ffi, finishlist, can_delay=False): - BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist, - can_delay) - if not can_delay: - self.finish_backend_type(ffi, finishlist) - return BType - - def finish_backend_type(self, ffi, finishlist): - if self.completed: - if self.completed != 2: - raise NotImplementedError("recursive structure declaration " - "for '%s'" % (self.name,)) - return - BType = ffi._cached_btypes[self] - # - self.completed = 1 - # - if self.fldtypes is None: - pass # not completing it: it's an opaque struct - # - elif self.fixedlayout is None: - fldtypes = [tp.get_cached_btype(ffi, finishlist) - for tp in self.fldtypes] - lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) - sflags = 0 - if self.packed: - sflags = 8 # SF_PACKED - ffi._backend.complete_struct_or_union(BType, lst, self, - -1, -1, sflags) - # - else: - fldtypes = [] - fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout - for i in range(len(self.fldnames)): - fsize = fieldsize[i] - ftype = self.fldtypes[i] - # - if isinstance(ftype, ArrayType) and ftype.length == '...': - # fix the length to match the total size - BItemType = ftype.item.get_cached_btype(ffi, finishlist) - nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) - if nrest != 0: - self._verification_error( - "field '%s.%s' has a bogus size?" % ( - self.name, self.fldnames[i] or '{}')) - ftype = ftype.resolve_length(nlen) - self.fldtypes = (self.fldtypes[:i] + (ftype,) + - self.fldtypes[i+1:]) - # - BFieldType = ftype.get_cached_btype(ffi, finishlist) - if isinstance(ftype, ArrayType) and ftype.length is None: - assert fsize == 0 - else: - bitemsize = ffi.sizeof(BFieldType) - if bitemsize != fsize: - self._verification_error( - "field '%s.%s' is declared as %d bytes, but is " - "really %d bytes" % (self.name, - self.fldnames[i] or '{}', - bitemsize, fsize)) - fldtypes.append(BFieldType) - # - lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) - ffi._backend.complete_struct_or_union(BType, lst, self, - totalsize, totalalignment) - self.completed = 2 - - def _verification_error(self, msg): - raise VerificationError(msg) - - def check_not_partial(self): - if self.partial and self.fixedlayout is None: - raise VerificationMissing(self._get_c_name()) - - def build_backend_type(self, ffi, finishlist): - self.check_not_partial() - finishlist.append(self) - # - return global_cache(self, ffi, 'new_%s_type' % self.kind, - self.get_official_name(), key=self) - - -class StructType(StructOrUnion): - kind = 'struct' - - -class UnionType(StructOrUnion): - kind = 'union' - - -class EnumType(StructOrUnionOrEnum): - kind = 'enum' - partial = False - partial_resolved = False - - def __init__(self, name, enumerators, enumvalues, baseinttype=None): - self.name = name - self.enumerators = enumerators - self.enumvalues = enumvalues - self.baseinttype = baseinttype - self.build_c_name_with_marker() - - def force_the_name(self, forcename): - StructOrUnionOrEnum.force_the_name(self, forcename) - if self.forcename is None: - name = self.get_official_name() - self.forcename = '$' + name.replace(' ', '_') - - def check_not_partial(self): - if self.partial and not self.partial_resolved: - raise VerificationMissing(self._get_c_name()) - - def build_backend_type(self, ffi, finishlist): - self.check_not_partial() - base_btype = self.build_baseinttype(ffi, finishlist) - return global_cache(self, ffi, 'new_enum_type', - self.get_official_name(), - self.enumerators, self.enumvalues, - base_btype, key=self) - - def build_baseinttype(self, ffi, finishlist): - if self.baseinttype is not None: - return self.baseinttype.get_cached_btype(ffi, finishlist) - # - if self.enumvalues: - smallest_value = min(self.enumvalues) - largest_value = max(self.enumvalues) - else: - import warnings - try: - # XXX! The goal is to ensure that the warnings.warn() - # will not suppress the warning. We want to get it - # several times if we reach this point several times. - __warningregistry__.clear() - except NameError: - pass - warnings.warn("%r has no values explicitly defined; " - "guessing that it is equivalent to 'unsigned int'" - % self._get_c_name()) - smallest_value = largest_value = 0 - if smallest_value < 0: # needs a signed type - sign = 1 - candidate1 = PrimitiveType("int") - candidate2 = PrimitiveType("long") - else: - sign = 0 - candidate1 = PrimitiveType("unsigned int") - candidate2 = PrimitiveType("unsigned long") - btype1 = candidate1.get_cached_btype(ffi, finishlist) - btype2 = candidate2.get_cached_btype(ffi, finishlist) - size1 = ffi.sizeof(btype1) - size2 = ffi.sizeof(btype2) - if (smallest_value >= ((-1) << (8*size1-1)) and - largest_value < (1 << (8*size1-sign))): - return btype1 - if (smallest_value >= ((-1) << (8*size2-1)) and - largest_value < (1 << (8*size2-sign))): - return btype2 - raise CDefError("%s values don't all fit into either 'long' " - "or 'unsigned long'" % self._get_c_name()) - -def unknown_type(name, structname=None): - if structname is None: - structname = '$%s' % name - tp = StructType(structname, None, None, None) - tp.force_the_name(name) - tp.origin = "unknown_type" - return tp - -def unknown_ptr_type(name, structname=None): - if structname is None: - structname = '$$%s' % name - tp = StructType(structname, None, None, None) - return NamedPointerType(tp, name) - - -def global_cache(srctype, ffi, funcname, *args, **kwds): - key = kwds.pop('key', (funcname, args)) - assert not kwds - try: - return ffi._backend.__typecache[key] - except KeyError: - pass - except AttributeError: - # initialize the __typecache attribute, either at the module level - # if ffi._backend is a module, or at the class level if ffi._backend - # is some instance. - if isinstance(ffi._backend, types.ModuleType): - ffi._backend.__typecache = weakref.WeakValueDictionary() - else: - type(ffi._backend).__typecache = weakref.WeakValueDictionary() - try: - res = getattr(ffi._backend, funcname)(*args) - except NotImplementedError as e: - raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) - # note that setdefault() on WeakValueDictionary is not atomic - # and contains a rare bug (http://bugs.python.org/issue19542); - # we have to use a lock and do it ourselves - cache = ffi._backend.__typecache - with global_lock: - res1 = cache.get(key) - if res1 is None: - cache[key] = res - return res - else: - return res1 - -def pointer_cache(ffi, BType): - return global_cache('?', ffi, 'new_pointer_type', BType) - -def attach_exception_info(e, name): - if e.args and type(e.args[0]) is str: - e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:] diff --git a/pypy/module/cpyext/commontypes.py b/pypy/module/cpyext/commontypes.py deleted file mode 100644 index fb2b7ff6a9..0000000000 --- a/pypy/module/cpyext/commontypes.py +++ /dev/null @@ -1,80 +0,0 @@ -import sys -from . import cmodel as model -from .error import FFIError - - -COMMON_TYPES = {} - -try: - # fetch "bool" and all simple Windows types - from _cffi_backend import _get_common_types - _get_common_types(COMMON_TYPES) -except ImportError: - pass - -COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') -COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above - -for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: - if _type.endswith('_t'): - COMMON_TYPES[_type] = _type -del _type - -_CACHE = {} - -def resolve_common_type(parser, commontype): - try: - return _CACHE[commontype] - except KeyError: - cdecl = COMMON_TYPES.get(commontype, commontype) - if not isinstance(cdecl, str): - result, quals = cdecl, 0 # cdecl is already a BaseType - elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: - result, quals = model.PrimitiveType(cdecl), 0 - elif cdecl == 'set-unicode-needed': - raise FFIError("The Windows type %r is only available after " - "you call ffi.set_unicode()" % (commontype,)) - else: - if commontype == cdecl: - raise FFIError( - "Unsupported type: %r. Please look at " - "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " - "and file an issue if you think this type should really " - "be supported." % (commontype,)) - result, quals = parser.parse_type_and_quals(cdecl) # recursive - - assert isinstance(result, model.BaseTypeByIdentity) - _CACHE[commontype] = result, quals - return result, quals - - -# ____________________________________________________________ -# extra types for Windows (most of them are in commontypes.c) - - -def win_common_types(): - return { - "UNICODE_STRING": model.StructType( - "_UNICODE_STRING", - ["Length", - "MaximumLength", - "Buffer"], - [model.PrimitiveType("unsigned short"), - model.PrimitiveType("unsigned short"), - model.PointerType(model.PrimitiveType("wchar_t"))], - [-1, -1, -1]), - "PUNICODE_STRING": "UNICODE_STRING *", - "PCUNICODE_STRING": "const UNICODE_STRING *", - - "TBYTE": "set-unicode-needed", - "TCHAR": "set-unicode-needed", - "LPCTSTR": "set-unicode-needed", - "PCTSTR": "set-unicode-needed", - "LPTSTR": "set-unicode-needed", - "PTSTR": "set-unicode-needed", - "PTBYTE": "set-unicode-needed", - "PTCHAR": "set-unicode-needed", - } - -if sys.platform == 'win32': - COMMON_TYPES.update(win_common_types()) diff --git a/pypy/module/cpyext/cparser.py b/pypy/module/cpyext/cparser.py deleted file mode 100644 index 3b4815fc6b..0000000000 --- a/pypy/module/cpyext/cparser.py +++ /dev/null @@ -1,907 +0,0 @@ -from collections import OrderedDict -from itertools import izip -from . import cmodel as model -from .commontypes import COMMON_TYPES, resolve_common_type -from .error import FFIError, CDefError -try: - from cffi import _pycparser as pycparser -except ImportError: - import pycparser -import weakref, re, sys -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rlib.rfile import FILEP -from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rfficache, rffi_platform -from rpython.flowspace.model import Constant, const -from rpython.flowspace.specialcase import register_flow_sc -from rpython.flowspace.flowcontext import FlowingError - -_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", - re.DOTALL | re.MULTILINE) -_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" - r"\b((?:[^\n\\]|\\.)*?)$", - re.DOTALL | re.MULTILINE) -_r_words = re.compile(r"\w+|\S") -_parser_cache = None -_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) -_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") -_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") -_r_cdecl = re.compile(r"\b__cdecl\b") -_r_star_const_space = re.compile( # matches "* const " - r"[*]\s*((const|volatile|restrict)\b\s*)+") - -def _get_parser(): - global _parser_cache - if _parser_cache is None: - _parser_cache = pycparser.CParser() - return _parser_cache - -def _preprocess(csource, macros): - # Remove comments. NOTE: this only work because the cdef() section - # should not contain any string literal! - csource = _r_comment.sub(' ', csource) - # Remove the "#define FOO x" lines - for match in _r_define.finditer(csource): - macroname, macrovalue = match.groups() - macrovalue = macrovalue.replace('\\\n', '').strip() - macros[macroname] = macrovalue - csource = _r_define.sub('', csource) - # - # BIG HACK: replace WINAPI or __stdcall with "volatile const". - # It doesn't make sense for the return type of a function to be - # "volatile volatile const", so we abuse it to detect __stdcall... - # Hack number 2 is that "int(volatile *fptr)();" is not valid C - # syntax, so we place the "volatile" before the opening parenthesis. - csource = _r_stdcall2.sub(' volatile volatile const(', csource) - csource = _r_stdcall1.sub(' volatile volatile const ', csource) - csource = _r_cdecl.sub(' ', csource) - - for name, value in reversed(macros.items()): - csource = re.sub(r'\b%s\b' % name, value, csource) - - return csource, macros - -def _common_type_names(csource): - # Look in the source for what looks like usages of types from the - # list of common types. A "usage" is approximated here as the - # appearance of the word, minus a "definition" of the type, which - # is the last word in a "typedef" statement. Approximative only - # but should be fine for all the common types. - look_for_words = set(COMMON_TYPES) - look_for_words.add(';') - look_for_words.add(',') - look_for_words.add('(') - look_for_words.add(')') - look_for_words.add('typedef') - words_used = set() - is_typedef = False - paren = 0 - previous_word = '' - for word in _r_words.findall(csource): - if word in look_for_words: - if word == ';': - if is_typedef: - words_used.discard(previous_word) - look_for_words.discard(previous_word) - is_typedef = False - elif word == 'typedef': - is_typedef = True - paren = 0 - elif word == '(': - paren += 1 - elif word == ')': - paren -= 1 - elif word == ',': - if is_typedef and paren == 0: - words_used.discard(previous_word) - look_for_words.discard(previous_word) - else: # word in COMMON_TYPES - words_used.add(word) - previous_word = word - return words_used - - -class Parser(object): - - def __init__(self): - self._declarations = OrderedDict() - self._included_declarations = set() - self._anonymous_counter = 0 - self._structnode2type = weakref.WeakKeyDictionary() - self._options = {} - self._int_constants = {} - self._recomplete = [] - self._macros = OrderedDict() - - def _parse(self, csource): - # modifies self._macros in-place - csource, macros = _preprocess(csource, self._macros) - # XXX: for more efficiency we would need to poke into the - # internals of CParser... the following registers the - # typedefs, because their presence or absence influences the - # parsing itself (but what they are typedef'ed to plays no role) - ctn = _common_type_names(csource) - typenames = [] - for name in sorted(self._declarations): - if name.startswith('typedef '): - name = name[8:] - typenames.append(name) - ctn.discard(name) - typenames += sorted(ctn) - # - csourcelines = ['typedef int %s;' % typename for typename in typenames] - csourcelines.append('typedef int __dotdotdot__;') - csourcelines.append(csource) - csource = '\n'.join(csourcelines) - try: - ast = _get_parser().parse(csource) - except pycparser.c_parser.ParseError as e: - self.convert_pycparser_error(e, csource) - # csource will be used to find buggy source text - return ast, macros, csource - - def _convert_pycparser_error(self, e, csource): - # xxx look for ":NUM:" at the start of str(e) and try to interpret - # it as a line number - line = None - msg = str(e) - if msg.startswith(':') and ':' in msg[1:]: - linenum = msg[1:msg.find(':',1)] - if linenum.isdigit(): - linenum = int(linenum, 10) - csourcelines = csource.splitlines() - if 1 <= linenum <= len(csourcelines): - line = csourcelines[linenum-1] - return line - - def convert_pycparser_error(self, e, csource): - line = self._convert_pycparser_error(e, csource) - - msg = str(e) - if line: - msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) - else: - msg = 'parse error\n%s' % (msg,) - raise CDefError(msg) - - def parse(self, csource, override=False, packed=False, dllexport=False): - prev_options = self._options - try: - self._options = {'override': override, - 'packed': packed, - 'dllexport': dllexport} - self._internal_parse(csource) - finally: - self._options = prev_options - - def _internal_parse(self, csource): - ast, macros, csource = self._parse(csource) - # add the macros - self._process_macros(macros) - # find the first "__dotdotdot__" and use that as a separator - # between the repeated typedefs and the real csource - iterator = iter(ast.ext) - for decl in iterator: - if decl.name == '__dotdotdot__': - break - # - try: - for decl in iterator: - if isinstance(decl, pycparser.c_ast.Decl): - self._parse_decl(decl) - elif isinstance(decl, pycparser.c_ast.Typedef): - self._parse_typedef(decl) - elif decl.__class__.__name__ == 'Pragma': - pass # skip pragma, only in pycparser 2.15 - else: - raise CDefError("unrecognized construct", decl) - except FFIError as e: - msg = self._convert_pycparser_error(e, csource) - if msg: - e.args = (e.args[0] + "\n *** Err: %s" % msg,) - raise - - def _add_constants(self, key, val): - if key in self._int_constants: - if self._int_constants[key] == val: - return # ignore identical double declarations - raise FFIError( - "multiple declarations of constant: %s" % (key,)) - self._int_constants[key] = val - - def _add_integer_constant(self, name, int_str): - int_str = int_str.lower().rstrip("ul") - neg = int_str.startswith('-') - if neg: - int_str = int_str[1:] - # "010" is not valid oct in py3 - if (int_str.startswith("0") and int_str != '0' - and not int_str.startswith("0x")): - int_str = "0o" + int_str[1:] - pyvalue = int(int_str, 0) - if neg: - pyvalue = -pyvalue - self._add_constants(name, pyvalue) - self._declare('macro ' + name, pyvalue) - - def _process_macros(self, macros): - for key, value in macros.items(): - value = value.strip() - if _r_int_literal.match(value): - self._add_integer_constant(key, value) - else: - self._declare('macro ' + key, value) - - def _declare_function(self, tp, quals, decl): - tp = self._get_type_pointer(tp, quals) - if self._options.get('dllexport'): - tag = 'dllexport_python ' - else: - tag = 'function ' - self._declare(tag + decl.name, tp) - - def _parse_typedef(self, decl): - if not decl.name: - raise CDefError("typedef does not declare any name", decl) - realtype, quals = self._get_type_and_quals( - decl.type, name=decl.name, partial_length_ok=True) - tp = model.DefinedType(decl.name, realtype, quals) - self._declare('typedef ' + decl.name, tp) - - def _parse_decl(self, decl): - node = decl.type - if isinstance(node, pycparser.c_ast.FuncDecl): - tp, quals = self._get_type_and_quals(node, name=decl.name) - assert isinstance(tp, model.RawFunctionType) - self._declare_function(tp, quals, decl) - else: - if isinstance(node, pycparser.c_ast.Struct): - self._get_struct_union_enum_type('struct', node) - elif isinstance(node, pycparser.c_ast.Union): - self._get_struct_union_enum_type('union', node) - elif isinstance(node, pycparser.c_ast.Enum): - self._get_struct_union_enum_type('enum', node) - elif not decl.name: - raise CDefError("construct does not declare any variable", - decl) - # - if decl.name: - tp, quals = self._get_type_and_quals(node, - partial_length_ok=True) - if tp.is_raw_function: - self._declare_function(tp, quals, decl) - elif (tp.is_integer_type() and - hasattr(decl, 'init') and - hasattr(decl.init, 'value') and - _r_int_literal.match(decl.init.value)): - self._add_integer_constant(decl.name, decl.init.value) - elif (tp.is_integer_type() and - isinstance(decl.init, pycparser.c_ast.UnaryOp) and - decl.init.op == '-' and - hasattr(decl.init.expr, 'value') and - _r_int_literal.match(decl.init.expr.value)): - self._add_integer_constant(decl.name, - '-' + decl.init.expr.value) - else: - if (quals & model.Q_CONST) and not tp.is_array_type: - self._declare('constant ' + decl.name, tp, quals=quals) - else: - self._declare('variable ' + decl.name, tp, quals=quals) - - def parse_type(self, cdecl): - return self.parse_type_and_quals(cdecl)[0] - - def parse_type_and_quals(self, cdecl): - ast, _, _ = self._parse('void __dummy(\n%s\n);' % cdecl) - exprnode = ast.ext[-1].type.args.params[0] - if isinstance(exprnode, pycparser.c_ast.ID): - raise CDefError("unknown identifier '%s'" % (exprnode.name,)) - return self._get_type_and_quals(exprnode.type) - - def _declare(self, name, obj, included=False, quals=0): - if name in self._declarations: - prevobj, prevquals = self._declarations[name] - if prevobj is obj and prevquals == quals: - return - self._declarations[name] = (obj, quals) - if included: - self._included_declarations.add(obj) - - def _extract_quals(self, type): - quals = 0 - if isinstance(type, (pycparser.c_ast.TypeDecl, - pycparser.c_ast.PtrDecl)): - if 'const' in type.quals: - quals |= model.Q_CONST - if 'volatile' in type.quals: - quals |= model.Q_VOLATILE - if 'restrict' in type.quals: - quals |= model.Q_RESTRICT - return quals - - def _get_type_pointer(self, type, quals, declname=None): - if isinstance(type, model.RawFunctionType): - return type.as_function_pointer() - if (isinstance(type, model.StructOrUnionOrEnum) and - type.name.startswith('$') and type.name[1:].isdigit() and - type.forcename is None and declname is not None): - return model.NamedPointerType(type, declname, quals) - return model.PointerType(type, quals) - - def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False): - if isinstance(typenode, pycparser.c_ast.ArrayDecl): - # array type - if typenode.dim is None: - length = None - else: - length = self._parse_constant( - typenode.dim, partial_length_ok=partial_length_ok) - tp, quals = self._get_type_and_quals(typenode.type, - partial_length_ok=partial_length_ok) - return model.ArrayType(tp, length), quals - # - if isinstance(typenode, pycparser.c_ast.PtrDecl): - # pointer type - itemtype, itemquals = self._get_type_and_quals(typenode.type) - tp = self._get_type_pointer(itemtype, itemquals, declname=name) - quals = self._extract_quals(typenode) - return tp, quals - # - if isinstance(typenode, pycparser.c_ast.TypeDecl): - quals = self._extract_quals(typenode) - type = typenode.type - if isinstance(type, pycparser.c_ast.IdentifierType): - # first, dereference typedefs, if we have it already parsed, we're good - if (len(type.names) == 1 and - ('typedef ' + type.names[0]) in self._declarations): - tp0, quals0 = self._declarations['typedef ' + type.names[0]] - return tp0, (quals | quals0) - # assume a primitive type. get it from .names, but reduce - # synonyms to a single chosen combination - names = list(type.names) - if names != ['signed', 'char']: # keep this unmodified - prefixes = {} - while names: - name = names[0] - if name in ('short', 'long', 'signed', 'unsigned'): - prefixes[name] = prefixes.get(name, 0) + 1 - del names[0] - else: - break - # ignore the 'signed' prefix below, and reorder the others - newnames = [] - for prefix in ('unsigned', 'short', 'long'): - for i in range(prefixes.get(prefix, 0)): - newnames.append(prefix) - if not names: - names = ['int'] # implicitly - if names == ['int']: # but kill it if 'short' or 'long' - if 'short' in prefixes or 'long' in prefixes: - names = [] - names = newnames + names - ident = ' '.join(names) - if ident == 'void': - return model.void_type, quals - tp0, quals0 = resolve_common_type(self, ident) - return tp0, (quals | quals0) - # - if isinstance(type, pycparser.c_ast.Struct): - # 'struct foobar' - tp = self._get_struct_union_enum_type('struct', type, name) - return tp, quals - # - if isinstance(type, pycparser.c_ast.Union): - # 'union foobar' - tp = self._get_struct_union_enum_type('union', type, name) - return tp, quals - # - if isinstance(type, pycparser.c_ast.Enum): - # 'enum foobar' - tp = self._get_struct_union_enum_type('enum', type, name) - return tp, quals - # - if isinstance(typenode, pycparser.c_ast.FuncDecl): - # a function type - return self._parse_function_type(typenode, name), 0 - # - # nested anonymous structs or unions end up here - if isinstance(typenode, pycparser.c_ast.Struct): - return self._get_struct_union_enum_type('struct', typenode, name, - nested=True), 0 - if isinstance(typenode, pycparser.c_ast.Union): - return self._get_struct_union_enum_type('union', typenode, name, - nested=True), 0 - # - raise FFIError(":%d: bad or unsupported type declaration" % - typenode.coord.line) - - def _parse_function_type(self, typenode, funcname=None): - params = list(getattr(typenode.args, 'params', [])) - for i, arg in enumerate(params): - if not hasattr(arg, 'type'): - raise CDefError("%s arg %d: unknown type '%s'" - " (if you meant to use the old C syntax of giving" - " untyped arguments, it is not supported)" - % (funcname or 'in expression', i + 1, - getattr(arg, 'name', '?'))) - ellipsis = ( - len(params) > 0 and - isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and - isinstance(params[-1].type.type, - pycparser.c_ast.IdentifierType) and - params[-1].type.type.names == ['__dotdotdot__']) - if ellipsis: - params.pop() - if not params: - raise CDefError( - "%s: a function with only '(...)' as argument" - " is not correct C" % (funcname or 'in expression')) - args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) - for argdeclnode in params] - if not ellipsis and args == [model.void_type]: - args = [] - result, quals = self._get_type_and_quals(typenode.type) - # the 'quals' on the result type are ignored. HACK: we absure them - # to detect __stdcall functions: we textually replace "__stdcall" - # with "volatile volatile const" above. - abi = None - if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway - if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: - abi = '__stdcall' - return model.RawFunctionType(tuple(args), result, ellipsis, abi) - - def _as_func_arg(self, type, quals): - if isinstance(type, model.ArrayType): - return model.PointerType(type.item, quals) - elif isinstance(type, model.RawFunctionType): - return type.as_function_pointer() - else: - return type - - def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): - # First, a level of caching on the exact 'type' node of the AST. - # This is obscure, but needed because pycparser "unrolls" declarations - # such as "typedef struct { } foo_t, *foo_p" and we end up with - # an AST that is not a tree, but a DAG, with the "type" node of the - # two branches foo_t and foo_p of the trees being the same node. - # It's a bit silly but detecting "DAG-ness" in the AST tree seems - # to be the only way to distinguish this case from two independent - # structs. See test_struct_with_two_usages. - try: - return self._structnode2type[type] - except KeyError: - pass - # - # Note that this must handle parsing "struct foo" any number of - # times and always return the same StructType object. Additionally, - # one of these times (not necessarily the first), the fields of - # the struct can be specified with "struct foo { ...fields... }". - # If no name is given, then we have to create a new anonymous struct - # with no caching; in this case, the fields are either specified - # right now or never. - # - force_name = name - name = type.name - # - # get the type or create it if needed - if name is None: - # 'force_name' is used to guess a more readable name for - # anonymous structs, for the common case "typedef struct { } foo". - if force_name is not None: - explicit_name = '$%s' % force_name - else: - self._anonymous_counter += 1 - explicit_name = '$%d' % self._anonymous_counter - tp = None - else: - explicit_name = name - key = '%s %s' % (kind, name) - tp, _ = self._declarations.get(key, (None, None)) - # - if tp is None: - if kind == 'struct': - tp = model.StructType(explicit_name, None, None, None) - elif kind == 'union': - tp = model.UnionType(explicit_name, None, None, None) - elif kind == 'enum': - tp = self._build_enum_type(explicit_name, type.values) - else: - raise AssertionError("kind = %r" % (kind,)) - if name is not None: - self._declare(key, tp) - else: - if kind == 'enum' and type.values is not None: - raise NotImplementedError( - "enum %s: the '{}' declaration should appear on the first " - "time the enum is mentioned, not later" % explicit_name) - if not tp.forcename: - tp.force_the_name(force_name) - if tp.forcename and '$' in tp.name: - self._declare('anonymous %s' % tp.forcename, tp) - # - self._structnode2type[type] = tp - # - # enums: done here - if kind == 'enum': - return tp - # - # is there a 'type.decls'? If yes, then this is the place in the - # C sources that declare the fields. If no, then just return the - # existing type, possibly still incomplete. - if type.decls is None: - return tp - # - if tp.fldnames is not None: - raise CDefError("duplicate declaration of struct %s" % name) - fldnames = [] - fldtypes = [] - fldbitsize = [] - fldquals = [] - for decl in type.decls: - if decl.bitsize is None: - bitsize = -1 - else: - bitsize = self._parse_constant(decl.bitsize) - self._partial_length = False - type, fqual = self._get_type_and_quals(decl.type, - partial_length_ok=True) - if self._partial_length: - self._make_partial(tp, nested) - if isinstance(type, model.StructType) and type.partial: - self._make_partial(tp, nested) - fldnames.append(decl.name or '') - fldtypes.append(type) - fldbitsize.append(bitsize) - fldquals.append(fqual) - tp.fldnames = tuple(fldnames) - tp.fldtypes = tuple(fldtypes) - tp.fldbitsize = tuple(fldbitsize) - tp.fldquals = tuple(fldquals) - if fldbitsize != [-1] * len(fldbitsize): - if isinstance(tp, model.StructType) and tp.partial: - raise NotImplementedError("%s: using both bitfields and '...;'" - % (tp,)) - tp.packed = self._options.get('packed') - if tp.completed: # must be re-completed: it is not opaque any more - tp.completed = 0 - self._recomplete.append(tp) - return tp - - def _make_partial(self, tp, nested): - if not isinstance(tp, model.StructOrUnion): - raise CDefError("%s cannot be partial" % (tp,)) - if not tp.has_c_name() and not nested: - raise NotImplementedError("%s is partial but has no C name" %(tp,)) - tp.partial = True - - def _parse_constant(self, exprnode, partial_length_ok=False): - # for now, limited to expressions that are an immediate number - # or positive/negative number - if isinstance(exprnode, pycparser.c_ast.Constant): - s = exprnode.value - if s.startswith('0'): - if s.startswith('0x') or s.startswith('0X'): - return int(s, 16) - return int(s, 8) - elif '1' <= s[0] <= '9': - return int(s, 10) - elif s[0] == "'" and s[-1] == "'" and ( - len(s) == 3 or (len(s) == 4 and s[1] == "\\")): - return ord(s[-2]) - else: - raise CDefError("invalid constant %r" % (s,)) - # - if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and - exprnode.op == '+'): - return self._parse_constant(exprnode.expr) - # - if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and - exprnode.op == '-'): - return -self._parse_constant(exprnode.expr) - # load previously defined int constant - if (isinstance(exprnode, pycparser.c_ast.ID) and - exprnode.name in self._int_constants): - return self._int_constants[exprnode.name] - # - if (isinstance(exprnode, pycparser.c_ast.ID) and - exprnode.name == '__dotdotdotarray__'): - if partial_length_ok: - self._partial_length = True - return '...' - raise FFIError(":%d: unsupported '[...]' here, cannot derive " - "the actual array length in this context" - % exprnode.coord.line) - # - raise FFIError(":%d: unsupported expression: expected a " - "simple numeric constant" % exprnode.coord.line) - - def _build_enum_type(self, explicit_name, decls): - if decls is not None: - partial = False - enumerators = [] - enumvalues = [] - nextenumvalue = 0 - for enum in decls.enumerators: - if enum.value is not None: - nextenumvalue = self._parse_constant(enum.value) - enumerators.append(enum.name) - enumvalues.append(nextenumvalue) - self._add_constants(enum.name, nextenumvalue) - nextenumvalue += 1 - enumerators = tuple(enumerators) - enumvalues = tuple(enumvalues) - tp = model.EnumType(explicit_name, enumerators, enumvalues) - tp.partial = partial - else: # opaque enum - tp = model.EnumType(explicit_name, (), ()) - return tp - - def include(self, other): - for name, (tp, quals) in other._declarations.items(): - if name.startswith('anonymous $enum_$'): - continue # fix for test_anonymous_enum_include - kind = name.split(' ', 1)[0] - if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef', 'macro'): - self._declare(name, tp, included=True, quals=quals) - for k, v in other._int_constants.items(): - self._add_constants(k, v) - for k, v in other._macros.items(): - self._macros[k] = v - -CNAME_TO_LLTYPE = { - 'char': rffi.CHAR, - 'double': rffi.DOUBLE, 'long double': rffi.LONGDOUBLE, - 'float': rffi.FLOAT, 'FILE': FILEP.TO} - -def add_inttypes(): - for name in rffi.TYPES: - if name.startswith('unsigned'): - rname = 'u' + name[9:] - else: - rname = name - rname = rname.replace(' ', '').upper() - CNAME_TO_LLTYPE[name] = rfficache.platform.types[rname] - -add_inttypes() -CNAME_TO_LLTYPE['int'] = rffi.INT_real -CNAME_TO_LLTYPE['wchar_t'] = lltype.UniChar -if 'ssize_t' not in CNAME_TO_LLTYPE: # on Windows - CNAME_TO_LLTYPE['ssize_t'] = CNAME_TO_LLTYPE['long'] - -def cname_to_lltype(name): - return CNAME_TO_LLTYPE[name] - -class DelayedStruct(object): - def __init__(self, name, fields, TYPE): - self.struct_name = name - self.type_name = None - self.fields = fields - self.TYPE = TYPE - - def get_type_name(self): - if self.type_name is not None: - return self.type_name - elif not self.struct_name.startswith('$'): - return 'struct %s' % self.struct_name - else: - raise ValueError('Anonymous struct') - - def __repr__(self): - return "<struct {struct_name}>".format(**vars(self)) - - -class CTypeSpace(object): - def __init__(self, parser=None, definitions=None, macros=None, - headers=None, includes=None): - self.definitions = definitions if definitions is not None else {} - self.macros = macros if macros is not None else {} - self.structs = {} - self.ctx = parser if parser else Parser() - self.headers = headers if headers is not None else ['sys/types.h'] - self.parsed_headers = [] - self.sources = [] - self._config_entries = OrderedDict() - self.includes = [] - self.struct_typedefs = {} - self._handled = set() - self._frozen = False - self._cdecl_type_cache = {} # {cdecl: TYPE} cache - if includes is not None: - for header in includes: - self.include(header) - - def include(self, other): - self.ctx.include(other.ctx) - self.structs.update(other.structs) - self.includes.append(other) - - def parse_source(self, source): - self.sources.append(source) - self.ctx.parse(source) - self.configure_types() - - def parse_header(self, header_path): - self.headers.append(str(header_path)) - self.parsed_headers.append(header_path) - self.ctx.parse(header_path.read()) - self.configure_types() - - def add_typedef(self, name, obj, quals): - assert name not in self.definitions - tp = self.convert_type(obj, quals) - if isinstance(tp, DelayedStruct): - if tp.type_name is None: - tp.type_name = name - tp = self.realize_struct(tp) - self.structs[obj.realtype] = tp - self.definitions[name] = tp - - def add_macro(self, name, value): - assert name not in self.macros - self.macros[name] = value - - def new_struct(self, obj): - if obj.name == '_IO_FILE': # cffi weirdness - return cname_to_lltype('FILE') - struct = DelayedStruct(obj.name, None, lltype.ForwardReference()) - # Cache it early, to avoid infinite recursion - self.structs[obj] = struct - if obj.fldtypes is not None: - struct.fields = zip( - obj.fldnames, - [self.convert_field(field) for field in obj.fldtypes]) - return struct - - def convert_field(self, obj): - tp = self.convert_type(obj) - if isinstance(tp, DelayedStruct): - tp = tp.TYPE - return tp - - def realize_struct(self, struct): - type_name = struct.get_type_name() - entry = rffi_platform.Struct(type_name, struct.fields) - self._config_entries[entry] = struct.TYPE - return struct.TYPE - - def build_eci(self): - all_sources = [] - for cts in self.includes: - all_sources.extend(cts.sources) - all_sources.extend(self.sources) - all_headers = self.headers - for x in self.includes: - for hdr in x.headers: - if hdr not in all_headers: - all_headers.append(hdr) - if sys.platform == 'win32': - compile_extra = ['-Dssize_t=long'] - else: - compile_extra = [] - return ExternalCompilationInfo( - post_include_bits=all_sources, includes=all_headers, - compile_extra=compile_extra) - - def configure_types(self): - for name, (obj, quals) in self.ctx._declarations.iteritems(): - if obj in self.ctx._included_declarations: - continue - if name in self._handled: - continue - self._handled.add(name) - if name.startswith('typedef '): - name = name[8:] - self.add_typedef(name, obj, quals) - elif name.startswith('macro '): - name = name[6:] - self.add_macro(name, obj) - if not self._config_entries: - return - eci = self.build_eci() - result = rffi_platform.configure_entries(list(self._config_entries), eci) - for entry, TYPE in izip(self._config_entries, result): - # hack: prevent the source from being pasted into common_header.h - del TYPE._hints['eci'] - self._config_entries[entry].become(TYPE) - self._config_entries.clear() - - def convert_type(self, obj, quals=0): - if isinstance(obj, model.DefinedType): - return self.convert_type(obj.realtype, obj.quals) - if isinstance(obj, model.PrimitiveType): - return cname_to_lltype(obj.name) - elif isinstance(obj, model.StructType): - if obj in self.structs: - return self.structs[obj] - return self.new_struct(obj) - elif isinstance(obj, model.PointerType): - TO = self.convert_type(obj.totype) - if TO is lltype.Void: - return rffi.VOIDP - elif isinstance(TO, DelayedStruct): - TO = TO.TYPE - if isinstance(TO, lltype.ContainerType): - return lltype.Ptr(TO) - else: - if obj.quals & model.Q_CONST: - return lltype.Ptr(lltype.Array( - TO, hints={'nolength': True, 'render_as_const': True})) - else: - return rffi.CArrayPtr(TO) - elif isinstance(obj, model.FunctionPtrType): - if obj.ellipsis: - raise NotImplementedError - args = [self.convert_type(arg) for arg in obj.args] - res = self.convert_type(obj.result) - return lltype.Ptr(lltype.FuncType(args, res)) - elif isinstance(obj, model.VoidType): - return lltype.Void - elif isinstance(obj, model.ArrayType): - return rffi.CFixedArray(self.convert_type(obj.item), obj.length) - else: - raise NotImplementedError - - def gettype(self, cdecl): - try: - return self._cdecl_type_cache[cdecl] - except KeyError: - result = self._real_gettype(cdecl) - self._cdecl_type_cache[cdecl] = result - return result - - def _real_gettype(self, cdecl): - obj = self.ctx.parse_type(cdecl) - result = self.convert_type(obj) - if isinstance(result, DelayedStruct): - result = result.TYPE - return result - - def cast(self, cdecl, value): - return rffi.cast(self.gettype(cdecl), value) - - def parse_func(self, cdecl): - cdecl = cdecl.strip() - if cdecl[-1] != ';': - cdecl += ';' - ast, _, _ = self.ctx._parse(cdecl) - decl = ast.ext[-1] - tp, quals = self.ctx._get_type_and_quals(decl.type, name=decl.name) - return FunctionDeclaration(decl.name, tp) - - def _freeze_(self): - if self._frozen: - return True - - @register_flow_sc(self.cast) - def sc_cast(ctx, v_decl, v_arg): - if not isinstance(v_decl, Constant): - raise FlowingError( - "The first argument of cts.cast() must be a constant.") - TP = self.gettype(v_decl.value) - return ctx.appcall(rffi.cast, const(TP), v_arg) - - @register_flow_sc(self.gettype) - def sc_gettype(ctx, v_decl): - if not isinstance(v_decl, Constant): - raise FlowingError( - "The argument of cts.gettype() must be a constant.") - return const(self.gettype(v_decl.value)) - - self._frozen = True - return True - -class FunctionDeclaration(object): - def __init__(self, name, tp): - self.name = name - self.tp = tp - - def get_llargs(self, cts): - return [cts.convert_type(arg) for arg in self.tp.args] - - def get_llresult(self, cts): - return cts.convert_type(self.tp.result) - -def parse_source(source, includes=None, headers=None, configure_now=True): - cts = CTypeSpace(headers=headers, includes=includes) - cts.parse_source(source) - return cts diff --git a/pypy/module/cpyext/error.py b/pypy/module/cpyext/error.py deleted file mode 100644 index 75a63d91cf..0000000000 --- a/pypy/module/cpyext/error.py +++ /dev/null @@ -1,20 +0,0 @@ - -class FFIError(Exception): - pass - -class CDefError(Exception): - def __str__(self): - try: - line = 'line %d: ' % (self.args[1].coord.line,) - except (AttributeError, TypeError, IndexError): - line = '' - return '%s%s' % (line, self.args[0]) - -class VerificationError(Exception): - """ An error raised when verification fails - """ - -class VerificationMissing(Exception): - """ An error raised when incomplete structures are passed into - cdef, but no verification has been done - """ diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py index a9bb200094..5e16133b94 100644 --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -64,7 +64,7 @@ def PyEval_GetFrame(space): caller = space.getexecutioncontext().gettopframe_nohidden() return caller # borrowed ref, may be null -@cpython_api([PyCodeObject, PyObject, PyObject], PyObject) +@cpython_api([PyObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): """This is a simplified interface to PyEval_EvalCodeEx(), with just the code object, and the dictionaries of global and local variables. diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h index b4f0945fab..006748d610 100644 --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -32,8 +32,8 @@ * module/sys/version.py * doc/conf.py */ -#define PYPY_VERSION "7.3.3-alpha0" -#define PYPY_VERSION_NUM 0x07030300 +#define PYPY_VERSION "7.3.4-alpha0" +#define PYPY_VERSION_NUM 0x07030400 /* Defined to mean a PyPy where cpyext holds more regular references to PyObjects, e.g. staying alive as long as the internal PyPy object stays alive. */ diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h index c6a0ba5df1..981c011783 100644 --- a/pypy/module/cpyext/include/unicodeobject.h +++ b/pypy/module/cpyext/include/unicodeobject.h @@ -17,7 +17,7 @@ PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs); PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...); #define PyUnicode_Check(op) \ - PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) + PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) #define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type) #ifdef __cplusplus diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py index 5cae2d920e..3a7cdaa2b2 100644 --- a/pypy/module/cpyext/memoryobject.py +++ b/pypy/module/cpyext/memoryobject.py @@ -1,5 +1,5 @@ from pypy.module.cpyext.api import ( - cpython_api, CANNOT_FAIL, Py_MAX_FMT, Py_MAX_NDIMS, build_type_checkers, + cpython_api, CANNOT_FAIL, Py_MAX_NDIMS, build_type_checkers, Py_ssize_tP, cts, parse_dir, bootstrap_function, Py_bufferP, slot_function, PyBUF_READ, PyBUF_WRITE) from pypy.module.cpyext.pyobject import ( @@ -18,6 +18,7 @@ PyMemoryViewObject = cts.gettype('PyMemoryViewObject*') PyMemoryView_Check, PyMemoryView_CheckExact = build_type_checkers("MemoryView") +FORMAT_ALLOCATED = 0x04 @bootstrap_function def init_memoryobject(space): @@ -87,9 +88,13 @@ def memory_realize(space, obj): @slot_function([PyObject], lltype.Void) def memory_dealloc(space, py_obj): mem_obj = rffi.cast(PyMemoryViewObject, py_obj) - if mem_obj.c_view.c_obj: - decref(space, mem_obj.c_view.c_obj) - mem_obj.c_view.c_obj = rffi.cast(PyObject, 0) + view = mem_obj.c_view + if view.c_obj: + decref(space, view.c_obj) + view.c_obj = rffi.cast(PyObject, 0) + flags = widen(view.c_flags) + if flags & FORMAT_ALLOCATED == FORMAT_ALLOCATED: + lltype.free(view.c_format, flavor='raw') _dealloc(space, py_obj) def fill_Py_buffer(space, buf, view): @@ -98,19 +103,13 @@ def fill_Py_buffer(space, buf, view): view.c_len = buf.getlength() view.c_itemsize = buf.getitemsize() rffi.setintfield(view, 'c_ndim', ndim) - view.c_format = rffi.cast(rffi.CCHARP, view.c__format) fmt = buf.getformat() - n = Py_MAX_FMT - 1 # NULL terminated buffer - if len(fmt) > n: - w_message = space.newbytes("PyPy specific Py_MAX_FMT is %d which is too " - "small for buffer format, %d needed" % ( - Py_MAX_FMT, len(fmt))) - w_stacklevel = space.newint(1) - w_module = PyImport_Import(space, space.newbytes("warnings")) - w_warn = space.getattr(w_module, space.newbytes("warn")) - space.call_function(w_warn, w_message, space.w_None, w_stacklevel) - else: - n = len(fmt) + n = len(fmt) + view.c_format = lltype.malloc(rffi.CCHARP.TO, n + 1, flavor='raw', + add_memory_pressure=True) + flags = widen(view.c_flags) + flags |= FORMAT_ALLOCATED + view.c_flags = rffi.cast(rffi.INT_real, flags) for i in range(n): view.c_format[i] = fmt[i] view.c_format[n] = '\x00' diff --git a/pypy/module/cpyext/parse/cpyext_descrobject.h b/pypy/module/cpyext/parse/cpyext_descrobject.h index fe361125f7..184c41af1d 100644 --- a/pypy/module/cpyext/parse/cpyext_descrobject.h +++ b/pypy/module/cpyext/parse/cpyext_descrobject.h @@ -1,3 +1,35 @@ +typedef PyObject *(*getter)(PyObject *, void *); +typedef int (*setter)(PyObject *, PyObject *, void *); + +typedef struct PyGetSetDef { + char *name; + getter get; + setter set; + char *doc; + void *closure; +} PyGetSetDef; + +typedef PyObject *(*wrapperfunc)(PyObject *self, PyObject *args, + void *wrapped); + +typedef PyObject *(*wrapperfunc_kwds)(PyObject *self, PyObject *args, + void *wrapped, PyObject *kwds); + +struct wrapperbase { + char *name; + int offset; + void *function; + wrapperfunc wrapper; + char *doc; + int flags; + PyObject *name_strobj; +}; + +/* Flags for above struct */ +#define PyWrapperFlag_KEYWORDS 1 /* wrapper function takes keyword args */ + +/* Various kinds of descriptor objects */ + #define PyDescr_COMMON \ PyObject_HEAD \ PyTypeObject *d_type; \ diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h index ab1526569c..ebda6dd253 100644 --- a/pypy/module/cpyext/parse/cpyext_object.h +++ b/pypy/module/cpyext/parse/cpyext_object.h @@ -1,6 +1,11 @@ #pragma once -typedef long Py_ssize_t; /* CPython defines it in pyport.h */ +/* CPython defines Py_ssize_t in pyport.h as intptr_t */ +#ifdef _WIN64 +typedef long long Py_ssize_t; +#else +typedef long Py_ssize_t; +#endif #define PyObject_HEAD \ Py_ssize_t ob_refcnt; \ @@ -70,7 +75,6 @@ typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **); /* Py3k buffer interface, adapted for PyPy */ /* XXX remove this constant, us a PyObject_VAR_HEAD instead */ #define Py_MAX_NDIMS 36 -#define Py_MAX_FMT 128 typedef struct bufferinfo { void *buf; PyObject *obj; /* owned reference */ @@ -85,13 +89,14 @@ typedef struct bufferinfo { Py_ssize_t *shape; Py_ssize_t *strides; Py_ssize_t *suboffsets; /* alway NULL for app-level objects*/ - unsigned char _format[Py_MAX_FMT]; + void *internal; /* always NULL for app-level objects */ + /* PyPy extensions */ + int flags; Py_ssize_t _strides[Py_MAX_NDIMS]; Py_ssize_t _shape[Py_MAX_NDIMS]; /* static store for shape and strides of mono-dimensional buffers. */ /* Py_ssize_t smalltable[2]; */ - void *internal; /* always NULL for app-level objects */ } Py_buffer; @@ -188,18 +193,6 @@ typedef struct { releasebufferproc bf_releasebuffer; } PyBufferProcs; -/* from descrobject.h */ -typedef PyObject *(*getter)(PyObject *, void *); -typedef int (*setter)(PyObject *, PyObject *, void *); - -typedef struct PyGetSetDef { - char *name; - getter get; - setter set; - char *doc; - void *closure; -} PyGetSetDef; - /* from methodobject.h */ typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, diff --git a/pypy/module/cpyext/pystrtod.py b/pypy/module/cpyext/pystrtod.py index 1f70b3e7d4..85864c60ba 100644 --- a/pypy/module/cpyext/pystrtod.py +++ b/pypy/module/cpyext/pystrtod.py @@ -1,6 +1,6 @@ import errno from pypy.interpreter.error import oefmt -from pypy.module.cpyext.api import cpython_api, CONST_STRING, INTP_real +from pypy.module.cpyext.api import cpython_api, INTP_real from pypy.module.cpyext.pyobject import PyObject from rpython.rlib import rdtoa from rpython.rlib import rfloat @@ -22,7 +22,7 @@ DOUBLE_TO_STRING_TYPES_MAP = { rfloat.DIST_NAN: Py_DTST_NAN } -@cpython_api([CONST_STRING, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) +@cpython_api([rffi.CONST_CCHARP, rffi.CCHARPP, PyObject], rffi.DOUBLE, error=-1.0) @jit.dont_look_inside # direct use of _get_errno() def PyOS_string_to_double(space, s, endptr, w_overflow_exception): """Convert a string s to a double, raising a Python @@ -63,6 +63,45 @@ def PyOS_string_to_double(space, s, endptr, w_overflow_exception): endpos = (rffi.cast(rffi.LONG, endptr[0]) - rffi.cast(rffi.LONG, s)) if endpos == 0 or (not user_endptr and not endptr[0][0] == '\0'): + low = rffi.constcharp2str(s).lower() + sz = 0 + if len(low) < 3: + pass + elif low[0] == '-': + if low.startswith('-infinity'): + result = -rfloat.INFINITY + sz = len("-infinity") + elif low.startswith("-inf"): + result = -rfloat.INFINITY + sz = 4 + elif low.startswith("-nan"): + result = -rfloat.NAN + sz = 4 + elif low[0] == '+': + if low.startswith("+infinity"): + result = rfloat.INFINITY + sz = len("+infinity") + elif low.startswith("+inf"): + result = rfloat.INFINITY + sz = 4 + elif low.startswith("+nan"): + result = rfloat.NAN + sz = 4 + elif low.startswith("infinity"): + result = rfloat.INFINITY + sz = len("infinity") + elif low.startswith("inf"): + result = rfloat.INFINITY + sz = 3 + elif low.startswith("nan"): + result = rfloat.NAN + sz = 3 + # result is set to 0.0 for a parse_error in dtoa.c + # if it changed, we must have sucessfully converted + if result != 0.0: + if endptr: + endptr[0] = rffi.cast(rffi.CCHARP, rffi.ptradd(s, sz)) + return result raise oefmt(space.w_ValueError, "invalid input at position %d", endpos) err = rffi.cast(lltype.Signed, rposix._get_errno()) diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py index ba0e0cdbf3..fe4e2b744d 100644 --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -822,6 +822,8 @@ def slot_from___buffer__(space, typedef, buff_fn): try: c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address()) c_view.c_obj = make_ref(space, w_obj) + if space.isinstance_w(w_obj, space.w_bytes): + rffi.setintfield(c_view, 'c_readonly', 1) except ValueError: s = buf.as_str() w_s = space.newbytes(s) diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py index 61d6e68749..2111c2ed3c 100644 --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,9 +1,28 @@ import os import pytest +import sys + +disabled = None +THIS_DIR = os.path.dirname(__file__) + +if sys.maxsize > 2**32 and sys.platform == 'win32': + # cpyext not yet supported on windows 64 bit + disabled = True + +def pytest_ignore_collect(path, config): + path = str(path) + if disabled: + if os.path.commonprefix([path, THIS_DIR]) == THIS_DIR: # workaround for bug in pytest<3.0.5 + return True + +def pytest_collect_file(path, parent): + if disabled: + # We end up here when calling py.test .../test_foo.py directly + # It's OK to kill the whole session with the following line + pytest.skip("cpyext not yet supported on windows 64 bit") def pytest_configure(config): if config.getoption('runappdirect') or config.getoption('direct_apptest'): - import sys import py from pypy import pypydir sys.path.append(str(py.path.local(pypydir) / 'tool' / 'cpyext')) diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py index c51e2728df..b614e74fcf 100644 --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -1,11 +1,13 @@ import py, pytest import contextlib from rpython.rtyper.lltypesystem import lltype +from rpython.translator.c.database import LowLevelDatabase +from rpython.tool.cparser import parse_source from pypy.interpreter.baseobjspace import W_Root from pypy.module.cpyext.state import State from pypy.module.cpyext.api import ( slot_function, cpython_api, copy_header_files, INTERPLEVEL_API, - Py_ssize_t, Py_ssize_tP, PyObject, cts) + Py_ssize_t, Py_ssize_tP, PyObject, cts, ApiFunction) from pypy.module.cpyext.test.test_cpyext import ( freeze_refcnts, LeakCheckingTest) from pypy.interpreter.error import OperationError @@ -107,3 +109,18 @@ def test_copy_header_files(tmpdir): check('Python.h') check('modsupport.h') check('pypy_decl.h') + +def test_write_func(): + db = LowLevelDatabase() + cdef = """ + typedef ssize_t Py_ssize_t; + """ + cts = parse_source(cdef) + cdecl = "Py_ssize_t * some_func(Py_ssize_t*)" + decl = cts.parse_func(cdecl) + api_function = ApiFunction( + decl.get_llargs(cts), decl.get_llresult(cts), lambda space, x: None, + cdecl=decl) + assert (api_function.get_api_decl('some_func', db) + == "PyAPI_FUNC(Py_ssize_t *) some_func(Py_ssize_t * arg0);") + diff --git a/pypy/module/cpyext/test/test_cparser.py b/pypy/module/cpyext/test/test_cparser.py deleted file mode 100644 index 1119135ffa..0000000000 --- a/pypy/module/cpyext/test/test_cparser.py +++ /dev/null @@ -1,280 +0,0 @@ -from rpython.flowspace.model import const -from rpython.flowspace.objspace import build_flow -from rpython.translator.simplify import simplify_graph -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.cpyext.cparser import parse_source, CTypeSpace - -def test_configure(): - decl = """ - typedef ssize_t Py_ssize_t; - - typedef struct { - Py_ssize_t ob_refcnt; - Py_ssize_t ob_pypy_link; - double ob_fval; - } TestFloatObject; - """ - cts = parse_source(decl) - TestFloatObject = cts.definitions['TestFloatObject'] - assert isinstance(TestFloatObject, lltype.Struct) - assert TestFloatObject.c_ob_refcnt == rffi.SSIZE_T - assert TestFloatObject.c_ob_pypy_link == rffi.SSIZE_T - assert TestFloatObject.c_ob_fval == rffi.DOUBLE - -def test_simple(): - decl = "typedef ssize_t Py_ssize_t;" - cts = parse_source(decl) - assert cts.definitions == {'Py_ssize_t': rffi.SSIZE_T} - -def test_macro(): - decl = """ - typedef ssize_t Py_ssize_t; - - #define PyObject_HEAD \ - Py_ssize_t ob_refcnt; \ - Py_ssize_t ob_pypy_link; \ - - typedef struct { - PyObject_HEAD - double ob_fval; - } PyFloatObject; - """ - cts = parse_source(decl) - assert 'PyFloatObject' in cts.definitions - assert 'PyObject_HEAD' in cts.macros - -def test_include(): - cdef1 = """ - typedef ssize_t Py_ssize_t; - - #define PyObject_HEAD \ - Py_ssize_t ob_refcnt; \ - Py_ssize_t ob_pypy_link; \ - - typedef struct { - char *name; - } Type; - """ - cdef2 = """ - typedef struct { - PyObject_HEAD - Py_ssize_t ob_foo; - Type *type; - } Object; - """ - cts1 = parse_source(cdef1) - Type = cts1.definitions['Type'] - assert isinstance(Type, lltype.Struct) - cts2 = parse_source(cdef2, includes=[cts1]) - assert 'Type' not in cts2.definitions - Object = cts2.definitions['Object'] - assert Object.c_type.TO is Type - -def test_multiple_sources(): - cdef1 = """ - typedef ssize_t Py_ssize_t; - - #define PyObject_HEAD \ - Py_ssize_t ob_refcnt; \ - Py_ssize_t ob_pypy_link; \ - - typedef struct { - char *name; - } Type; - """ - cdef2 = """ - typedef struct { - PyObject_HEAD - Py_ssize_t ob_foo; - Type *type; - } Object; - """ - cts = CTypeSpace() - cts.parse_source(cdef1) - Type = cts.definitions['Type'] - assert isinstance(Type, lltype.Struct) - assert 'Object' not in cts.definitions - cts.parse_source(cdef2) - Object = cts.definitions['Object'] - assert Object.c_type.TO is Type - -def test_incomplete(): - cdef = """ - typedef ssize_t Py_ssize_t; - - typedef struct { - Py_ssize_t ob_refcnt; - Py_ssize_t ob_pypy_link; - struct _typeobject *ob_type; - } Object; - - typedef struct { - void *buf; - Object *obj; - } Buffer; - - """ - cts = parse_source(cdef) - Object = cts.gettype('Object') - assert isinstance(Object, lltype.Struct) - -def test_incomplete_struct(): - cdef = """ - typedef struct s *ptr; - struct s {void* x;}; - typedef struct s __s; // HACK! - """ - cts = parse_source(cdef) - PTR = cts.gettype("ptr") - assert isinstance(PTR.TO, lltype.Struct) - hash(PTR) - -def test_recursive(): - cdef = """ - typedef ssize_t Py_ssize_t; - - typedef struct { - Py_ssize_t ob_refcnt; - Py_ssize_t ob_pypy_link; - struct _typeobject *ob_type; - } Object; - - typedef struct { - void *buf; - Object *obj; - } Buffer; - - typedef struct _typeobject { - Object *obj; - } Type; - """ - cts = parse_source(cdef) - Object = cts.definitions['Object'] - assert isinstance(Object, lltype.Struct) - hash(Object) - -def test_nested_struct(): - cdef = """ - typedef struct { - int x; - } foo; - typedef struct { - foo y; - } bar; - """ - cts = parse_source(cdef) - bar = cts.gettype('bar') - assert isinstance(bar, lltype.Struct) - hash(bar) # bar is hashable - -def test_const(): - cdef = """ - typedef struct { - const char * const foo; - } bar; - """ - cts = parse_source(cdef) - assert cts.definitions['bar'].c_foo == rffi.CONST_CCHARP != rffi.CCHARP - -def test_gettype(): - decl = """ - typedef ssize_t Py_ssize_t; - - #define PyObject_HEAD \ - Py_ssize_t ob_refcnt; \ - Py_ssize_t ob_pypy_link; \ - - typedef struct { - PyObject_HEAD - double ob_fval; - } TestFloatObject; - """ - cts = parse_source(decl) - assert cts.gettype('Py_ssize_t') == rffi.SSIZE_T - assert cts.gettype('TestFloatObject *').TO.c_ob_refcnt == rffi.SSIZE_T - assert cts.cast('Py_ssize_t', 42) == rffi.cast(rffi.SSIZE_T, 42) - -def test_parse_funcdecl(): - decl = """ - typedef ssize_t Py_ssize_t; - - #define PyObject_HEAD \ - Py_ssize_t ob_refcnt; \ - Py_ssize_t ob_pypy_link; \ - - typedef struct { - PyObject_HEAD - double ob_fval; - } TestFloatObject; - - typedef TestFloatObject* (*func_t)(int, int); - """ - cts = parse_source(decl) - func_decl = cts.parse_func("func_t * some_func(TestFloatObject*)") - assert func_decl.name == 'some_func' - assert func_decl.get_llresult(cts) == cts.gettype('func_t*') - assert func_decl.get_llargs(cts) == [cts.gettype('TestFloatObject *')] - -def test_struct_in_func_args(): - decl = """ - typedef struct {int x;} obj; - typedef int (*func)(obj x); - """ - cts = parse_source(decl) - OBJ = cts.gettype('obj') - FUNCPTR = cts.gettype('func') - assert FUNCPTR.TO.ARGS == (OBJ,) - -def test_write_func(): - from ..api import ApiFunction - from rpython.translator.c.database import LowLevelDatabase - db = LowLevelDatabase() - cdef = """ - typedef ssize_t Py_ssize_t; - """ - cts = parse_source(cdef) - cdecl = "Py_ssize_t * some_func(Py_ssize_t*)" - decl = cts.parse_func(cdecl) - api_function = ApiFunction( - decl.get_llargs(cts), decl.get_llresult(cts), lambda space, x: None, - cdecl=decl) - assert (api_function.get_api_decl('some_func', db) == - "PyAPI_FUNC(Py_ssize_t *) some_func(Py_ssize_t * arg0);") - - -def test_wchar_t(): - cdef = """ - typedef struct { wchar_t* x; } test; - """ - cts = parse_source(cdef, headers=['stddef.h']) - obj = lltype.malloc(cts.gettype('test'), flavor='raw') - obj.c_x = cts.cast('wchar_t*', 0) - obj.c_x = lltype.nullptr(rffi.CWCHARP.TO) - lltype.free(obj, flavor='raw') - -def test_translate_cast(): - cdef = "typedef ssize_t Py_ssize_t;" - cts = parse_source(cdef) - - def f(): - return cts.cast('Py_ssize_t*', 0) - graph = build_flow(f) - simplify_graph(graph) - assert len(graph.startblock.operations) == 1 - op = graph.startblock.operations[0] - assert op.args[0] == const(rffi.cast) - assert op.args[1].value is cts.gettype('Py_ssize_t*') - -def test_translate_gettype(): - cdef = "typedef ssize_t Py_ssize_t;" - cts = parse_source(cdef) - - def f(): - return cts.gettype('Py_ssize_t*') - graph = build_flow(f) - simplify_graph(graph) - # Check that the result is constant-folded - assert graph.startblock.operations == [] - [link] = graph.startblock.exits - assert link.target is graph.returnblock - assert link.args[0] == const(rffi.CArrayPtr(rffi.SSIZE_T)) diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py index ea3789fd01..cfcfa04f51 100644 --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -185,12 +185,18 @@ class AppTestFloatMacros(AppTestCpythonExtensionBase): assert module.test() == float('inf') def test_Py_NAN(self): + import sys module = self.import_extension('foo', [ ("test", "METH_NOARGS", """ return PyFloat_FromDouble(Py_NAN); """), ]) - import struct - float_bits = struct.Struct('d').pack - assert float_bits(module.test()) == float_bits(float('nan')) + if sys.platform == 'win32': + # CPython does not enforce bit-compatibility between the NANs + import math + assert math.isnan(module.test()) + else: + import struct + float_bits = struct.Struct('d').pack + assert float_bits(module.test()) == float_bits(float('nan')) diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py index 026a363e92..d012b3f370 100644 --- a/pypy/module/cpyext/test/test_memoryobject.py +++ b/pypy/module/cpyext/test/test_memoryobject.py @@ -164,7 +164,19 @@ class AppTestBufferProtocol(AppTestCpythonExtensionBase): ("get_contiguous", "METH_O", """ return PyMemoryView_GetContiguous(args, PyBUF_READ, 'C'); - """) + """), + ("get_readonly", "METH_O", + """ + Py_buffer view; + int readonly; + memset(&view, 0, sizeof(view)); + if (PyObject_GetBuffer(args, &view, PyBUF_SIMPLE) != 0) { + return NULL; + } + readonly = view.readonly; + PyBuffer_Release(&view); + return PyLong_FromLong(readonly); + """), ]) module = self.import_module(name='buffer_test') arr = module.PyMyArray(10) @@ -177,6 +189,8 @@ class AppTestBufferProtocol(AppTestCpythonExtensionBase): foo.test_contiguous(arr) contig = foo.get_contiguous(arr) foo.test_contiguous(contig) + ro = foo.get_readonly(b'abc') + assert ro == 1 try: from _numpypy import multiarray as np except ImportError: @@ -289,22 +303,8 @@ class AppTestBufferInfo(AppTestCpythonExtensionBase): ('c', 'i')], ) x = np.arange(dt1.itemsize, dtype='int8').view(dt1) - # pytest can catch warnings from v2.8 and up, we ship 2.5 - import warnings - warnings.filterwarnings("error") - try: - try: - y = get_buffer_info(x, ['SIMPLE']) - except UserWarning as e: - pass - else: - assert False ,"PyPy-specific UserWarning not raised" \ - " on too long format string" - finally: - warnings.resetwarnings() # calling get_buffer_info on x creates a memory leak, # which is detected as an error at test teardown: # Exception TypeError: "'NoneType' object is not callable" # in <bound method ConcreteArray.__del__ ...> ignored - diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py index 445c473d85..e4f1dd575b 100644 --- a/pypy/module/cpyext/test/test_pystrtod.py +++ b/pypy/module/cpyext/test/test_pystrtod.py @@ -10,28 +10,28 @@ from pypy.module.cpyext.pystrtod import PyOS_string_to_double, INTP_real class TestPyOS_string_to_double(BaseApiTest): def test_simple_float(self, space): - s = rffi.str2charp('0.4') + s = rffi.str2constcharp('0.4') null = lltype.nullptr(rffi.CCHARPP.TO) r = PyOS_string_to_double(space, s, null, None) assert r == 0.4 rffi.free_charp(s) def test_empty_string(self, space): - s = rffi.str2charp('') + s = rffi.str2constcharp('') null = lltype.nullptr(rffi.CCHARPP.TO) with raises_w(space, ValueError): PyOS_string_to_double(space, s, null, None) rffi.free_charp(s) def test_bad_string(self, space): - s = rffi.str2charp(' 0.4') + s = rffi.str2constcharp(' 0.4') null = lltype.nullptr(rffi.CCHARPP.TO) with raises_w(space, ValueError): PyOS_string_to_double(space, s, null, None) rffi.free_charp(s) def test_overflow_pos(self, space): - s = rffi.str2charp('1e500') + s = rffi.str2constcharp('1e500') null = lltype.nullptr(rffi.CCHARPP.TO) r = PyOS_string_to_double(space, s, null, None) assert math.isinf(r) @@ -39,7 +39,7 @@ class TestPyOS_string_to_double(BaseApiTest): rffi.free_charp(s) def test_overflow_neg(self, space): - s = rffi.str2charp('-1e500') + s = rffi.str2constcharp('-1e500') null = lltype.nullptr(rffi.CCHARPP.TO) r = PyOS_string_to_double(space, s, null, None) assert math.isinf(r) @@ -47,14 +47,14 @@ class TestPyOS_string_to_double(BaseApiTest): rffi.free_charp(s) def test_overflow_exc(self, space): - s = rffi.str2charp('1e500') + s = rffi.str2constcharp('1e500') null = lltype.nullptr(rffi.CCHARPP.TO) with raises_w(space, ValueError): PyOS_string_to_double(space, s, null, space.w_ValueError) rffi.free_charp(s) def test_endptr_number(self, space): - s = rffi.str2charp('0.4') + s = rffi.str2constcharp('0.4') endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') r = PyOS_string_to_double(space, s, endp, None) assert r == 0.4 @@ -65,7 +65,7 @@ class TestPyOS_string_to_double(BaseApiTest): lltype.free(endp, flavor='raw') def test_endptr_tail(self, space): - s = rffi.str2charp('0.4 foo') + s = rffi.str2constcharp('0.4 foo') endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') r = PyOS_string_to_double(space, s, endp, None) assert r == 0.4 @@ -76,7 +76,7 @@ class TestPyOS_string_to_double(BaseApiTest): lltype.free(endp, flavor='raw') def test_endptr_no_conversion(self, space): - s = rffi.str2charp('foo') + s = rffi.str2constcharp('foo') endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') with raises_w(space, ValueError): PyOS_string_to_double(space, s, endp, None) @@ -86,6 +86,25 @@ class TestPyOS_string_to_double(BaseApiTest): rffi.free_charp(s) lltype.free(endp, flavor='raw') + def test_endptr_inf(self, space): + endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') + for test in ('inf', '+infinity', 'INF'): + s = rffi.str2constcharp(test) + r = PyOS_string_to_double(space, s, endp, None) + assert r == float('inf') + endp_addr = rffi.cast(rffi.LONG, endp[0]) + s_addr = rffi.cast(rffi.LONG, s) + assert endp_addr == s_addr + len(test) + rffi.free_charp(s) + s = rffi.str2constcharp('inf aaa') + r = PyOS_string_to_double(space, s, endp, None) + assert r == float('inf') + endp_addr = rffi.cast(rffi.LONG, endp[0]) + s_addr = rffi.cast(rffi.LONG, s) + # CPython returns 3 + assert endp_addr == s_addr + 3 + rffi.free_charp(s) + lltype.free(endp, flavor='raw') class TestPyOS_double_to_string(BaseApiTest): diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py index 5300214b05..73fefaf466 100644 --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -229,9 +229,10 @@ class TestUnicode(BaseApiTest): encoded_obj = PyUnicode_AsEncodedObject(space, space.wrap(u'sp�m'), utf_8, None) assert space.eq_w(encoded, encoded_obj) + one = space.newint(1) with raises_w(space, TypeError): PyUnicode_AsEncodedString( - space, space.newtuple([1, 2, 3]), None, None) + space, space.newtuple([one, one, one]), None, None) with raises_w(space, TypeError): PyUnicode_AsEncodedString(space, space.wrap(''), None, None) ascii = rffi.str2charp('ascii') diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py index 9cdce3b026..55224fc2c4 100644 --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -42,8 +42,6 @@ from pypy.module.cpyext.typeobjectdefs import ( from pypy.objspace.std.typeobject import W_TypeObject, find_best_base -cts.parse_header(parse_dir / "cpyext_descrobject.h") - #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type") @@ -75,7 +73,8 @@ class W_GetSetPropertyEx(GetSetProperty): self.name, self.w_type) -def PyDescr_NewGetSet(space, getset, w_type): +def PyDescr_NewGetSet(space, w_type, getset): + # Note the arguments are reversed return W_GetSetPropertyEx(getset, w_type) def make_GetSet(space, getsetprop): @@ -243,7 +242,7 @@ def convert_getset_defs(space, dict_w, getsets, w_type): if not name: break name = rffi.charp2str(name) - w_descr = PyDescr_NewGetSet(space, getset, w_type) + w_descr = PyDescr_NewGetSet(space, w_type, getset) dict_w[name] = w_descr def convert_member_defs(space, dict_w, members, w_type): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py index 31a6741cf1..9b92df2d05 100644 --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -1,5 +1,6 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import rstring +from rpython.rlib.rarithmetic import widen from rpython.tool.sourcetools import func_renamer from pypy.interpreter.error import OperationError, oefmt @@ -8,7 +9,7 @@ from pypy.interpreter.unicodehelper import ( unicode_encode_decimal) from pypy.module.unicodedata.interp_ucd import unicodedb from pypy.module.cpyext.api import ( - CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api, + CANNOT_FAIL, Py_ssize_t, Py_TPFLAGS_UNICODE_SUBCLASS, cpython_api, bootstrap_function, CONST_STRING, INTP_real, CONST_WSTRING, slot_function, cts, parse_dir) from pypy.module.cpyext.pyerrors import PyErr_BadArgument @@ -40,8 +41,11 @@ DEFAULT_ENCODING_SIZE = 100 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE, flavor='raw', zero=True) -PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers_flags("Unicode") +def PyUnicode_Check(ref): + if not ref: + return False + return (widen(ref.c_ob_type.c_tp_flags) & Py_TPFLAGS_UNICODE_SUBCLASS) != 0 def new_empty_unicode(space, length): """ @@ -228,9 +232,7 @@ def PyUnicode_AS_UNICODE(space, ref): def PyUnicode_AsUnicode(space, ref): """Return a read-only pointer to the Unicode object's internal Py_UNICODE buffer, NULL if unicode is not a Unicode object.""" - # Don't use PyUnicode_Check, it will realize the object :-( - w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) - if not space.issubtype_w(w_type, space.w_unicode): + if not PyUnicode_Check(ref): raise oefmt(space.w_TypeError, "expected unicode object") return PyUnicode_AS_UNICODE(space, rffi.cast(rffi.VOIDP, ref)) @@ -292,14 +294,8 @@ def PyUnicode_SetDefaultEncoding(space, encoding): default_encoding[0] = '\x00' return 0 -@cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) -def PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors): - """Encode a Unicode object and return the result as Python object. - encoding and errors have the same meaning as the parameters of the same name - in the Unicode encode() method. The codec to be used is looked up using - the Python codec registry. Return NULL if an exception was raised by the - codec.""" - if not PyUnicode_Check(space, w_unicode): +def _unicode_as_encoded_object(space, pyobj, llencoding, llerrors): + if not PyUnicode_Check(pyobj): PyErr_BadArgument(space) encoding = errors = None @@ -307,29 +303,40 @@ def PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors): encoding = rffi.charp2str(llencoding) if llerrors: errors = rffi.charp2str(llerrors) + w_unicode = from_ref(space, pyobj) return unicodeobject.encode_object(space, w_unicode, encoding, errors) @cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) -def PyUnicode_AsEncodedString(space, w_unicode, llencoding, llerrors): +def PyUnicode_AsEncodedObject(space, pyobj, llencoding, llerrors): + """Encode a Unicode object and return the result as Python object. + encoding and errors have the same meaning as the parameters of the same name + in the Unicode encode() method. The codec to be used is looked up using + the Python codec registry. Return NULL if an exception was raised by the + codec.""" + return _unicode_as_encoded_object(space, pyobj, llencoding, llerrors) + +@cpython_api([PyObject, CONST_STRING, CONST_STRING], PyObject) +def PyUnicode_AsEncodedString(space, pyref, llencoding, llerrors): """Encode a Unicode object and return the result as Python string object. encoding and errors have the same meaning as the parameters of the same name in the Unicode encode() method. The codec to be used is looked up using the Python codec registry. Return NULL if an exception was raised by the codec.""" - w_str = PyUnicode_AsEncodedObject(space, w_unicode, llencoding, llerrors) + w_str = _unicode_as_encoded_object(space, pyref, llencoding, llerrors) if not PyString_Check(space, w_str): raise oefmt(space.w_TypeError, "encoder did not return a string object") return w_str @cpython_api([PyObject], PyObject) -def PyUnicode_AsUnicodeEscapeString(space, w_unicode): +def PyUnicode_AsUnicodeEscapeString(space, pyobj): """Encode a Unicode object using Unicode-Escape and return the result as Python string object. Error handling is "strict". Return NULL if an exception was raised by the codec.""" - if not PyUnicode_Check(space, w_unicode): + if not PyUnicode_Check(pyobj): PyErr_BadArgument(space) + w_unicode = from_ref(space, pyobj) return unicodeobject.encode_object(space, w_unicode, 'unicode-escape', 'strict') @cpython_api([CONST_WSTRING, Py_ssize_t], PyObject, result_is_ll=True) @@ -470,12 +477,13 @@ def PyUnicode_Resize(space, ref, newsize): def make_conversion_functions(suffix, encoding, only_for_asstring=False): @cpython_api([PyObject], PyObject) @func_renamer('PyUnicode_As%sString' % suffix) - def PyUnicode_AsXXXString(space, w_unicode): + def PyUnicode_AsXXXString(space, pyobj): """Encode a Unicode object and return the result as Python string object. Error handling is "strict". Return NULL if an exception was raised by the codec.""" - if not PyUnicode_Check(space, w_unicode): + if not PyUnicode_Check(pyobj): PyErr_BadArgument(space) + w_unicode = from_ref(space, pyobj) return unicodeobject.encode_object(space, w_unicode, encoding, "strict") globals()['PyUnicode_As%sString' % suffix] = PyUnicode_AsXXXString diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c index 0b621e207f..00e2d62bb6 100644 --- a/pypy/module/faulthandler/faulthandler.c +++ b/pypy/module/faulthandler/faulthandler.c @@ -92,12 +92,12 @@ void pypy_faulthandler_write_uint(int fd, unsigned long uvalue, int min_digits) pypy_faulthandler_write(fd, p); } -static void pypy_faulthandler_write_hex(int fd, unsigned long uvalue) +static void pypy_faulthandler_write_hex(int fd, Unsigned uvalue) { char buf[48], *p = buf + 48; *--p = 0; do { - unsigned long byte = uvalue % 16UL; + Unsigned byte = uvalue % 16UL; assert(p > buf); if (byte < 10) *--p = '0' + byte; @@ -140,7 +140,7 @@ void pypy_faulthandler_dump_traceback(int fd, int all_threads, pypy_faulthandler_write(fd, my == p ? "Current thread 0x" : "Thread 0x"); - pypy_faulthandler_write_hex(fd, (unsigned long)p->thread_ident); + pypy_faulthandler_write_hex(fd, (Unsigned)p->thread_ident); pypy_faulthandler_write(fd, " (most recent call first," " approximate line numbers):\n"); diff --git a/pypy/module/micronumpy/test/conftest.py b/pypy/module/micronumpy/test/conftest.py new file mode 100644 index 0000000000..c07aa9eca4 --- /dev/null +++ b/pypy/module/micronumpy/test/conftest.py @@ -0,0 +1,22 @@ +import os +import pytest +import sys + +disabled = None +THIS_DIR = os.path.dirname(__file__) + +if sys.maxsize > 2**32 and sys.platform == 'win32': + # cpyext not yet supported on windows 64 bit + disabled = True + +def pytest_ignore_collect(path, config): + path = str(path) + if disabled: + if os.path.commonprefix([path, THIS_DIR]) == THIS_DIR: # workaround for bug in pytest<3.0.5 + return True + +def pytest_collect_file(path, parent): + if disabled: + # We end up here when calling py.test .../test_foo.py directly + # It's OK to kill the whole session with the following line + pytest.skip("cpyext not yet supported on windows 64 bit") diff --git a/pypy/module/operator/tscmp.c b/pypy/module/operator/tscmp.c index b68db6d2c9..8ec3dcdd4e 100644 --- a/pypy/module/operator/tscmp.c +++ b/pypy/module/operator/tscmp.c @@ -7,16 +7,16 @@ #include "tscmp.h" int -pypy_tscmp(const char *a, const char *b, long len_a, long len_b) +pypy_tscmp(const char *a, const char *b, Signed len_a, Signed len_b) { /* The volatile type declarations make sure that the compiler has no * chance to optimize and fold the code in any way that may change * the timing. */ - volatile long length; + volatile Signed length; volatile const char *left; volatile const char *right; - long i; + Signed i; char result; /* loop count depends on length of b */ @@ -44,16 +44,16 @@ pypy_tscmp(const char *a, const char *b, long len_a, long len_b) } int -pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, long len_a, long len_b) +pypy_tscmp_wide(const wchar_t *a, const wchar_t *b, Signed len_a, Signed len_b) { /* The volatile type declarations make sure that the compiler has no * chance to optimize and fold the code in any way that may change * the timing. */ - volatile long length; + volatile Signed length; volatile const wchar_t *left; volatile const wchar_t *right; - long i; + Signed i; wchar_t result; /* loop count depends on length of b */ diff --git a/pypy/module/operator/tscmp.h b/pypy/module/operator/tscmp.h index 4652aa0699..85a16a67f2 100644 --- a/pypy/module/operator/tscmp.h +++ b/pypy/module/operator/tscmp.h @@ -1,2 +1,2 @@ -RPY_EXTERN int pypy_tscmp(const char *, const char *, long, long); -RPY_EXTERN int pypy_tscmp_wide(const wchar_t *, const wchar_t *, long, long); +RPY_EXTERN int pypy_tscmp(const char *, const char *, Signed, Signed); +RPY_EXTERN int pypy_tscmp_wide(const wchar_t *, const wchar_t *, Signed, Signed); diff --git a/pypy/module/operator/tscmp.py b/pypy/module/operator/tscmp.py index b771f81884..1e08c21450 100644 --- a/pypy/module/operator/tscmp.py +++ b/pypy/module/operator/tscmp.py @@ -26,11 +26,11 @@ def llexternal(*args, **kwargs): pypy_tscmp = llexternal( 'pypy_tscmp', - [rffi.CCHARP, rffi.CCHARP, rffi.LONG, rffi.LONG], + [rffi.CCHARP, rffi.CCHARP, rffi.SIGNED, rffi.SIGNED], rffi.INT) pypy_tscmp_wide = llexternal( 'pypy_tscmp_wide', - [rffi.CWCHARP, rffi.CWCHARP, rffi.LONG, rffi.LONG], + [rffi.CWCHARP, rffi.CWCHARP, rffi.SIGNED, rffi.SIGNED], rffi.INT) diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py index e558f8c7fa..68381ac8d5 100644 --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -30,6 +30,7 @@ class TestCheckSignals: assert not space.is_true(w_received) # # calling ec.checksignals() should call it + print(space.getexecutioncontext().checksignals) space.getexecutioncontext().checksignals() assert space.is_true(w_received) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py index ff2813af1a..5b87289e41 100644 --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -12,7 +12,7 @@ CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h # make sure to keep PYPY_VERSION in sync with: # module/cpyext/include/patchlevel.h # doc/conf.py -PYPY_VERSION = (7, 3, 3, "alpha", 0) +PYPY_VERSION = (7, 3, 4, "alpha", 0) import pypy diff --git a/pypy/module/test_lib_pypy/test_posix_extra.py b/pypy/module/test_lib_pypy/test_posix_extra.py deleted file mode 100644 index fe58ec0a7e..0000000000 --- a/pypy/module/test_lib_pypy/test_posix_extra.py +++ /dev/null @@ -1,40 +0,0 @@ -import py -import sys, os, subprocess - - -CODE = """ -import sys, os, thread, time - -fd1, fd2 = os.pipe() -f1 = os.fdopen(fd1, 'r', 0) -f2 = os.fdopen(fd2, 'w', 0) - -def f(): - print "thread started" - x = f1.read(1) - assert x == "X" - print "thread exit" - -thread.start_new_thread(f, ()) -time.sleep(0.5) -if os.fork() == 0: # in the child - time.sleep(0.5) - x = f1.read(1) - assert x == "Y" - print "ok!" - sys.exit() - -f2.write("X") # in the parent -f2.write("Y") # in the parent -time.sleep(1.0) -""" - - -def test_thread_fork_file_lock(): - if not hasattr(os, 'fork'): - py.test.skip("requires 'fork'") - output = subprocess.check_output([sys.executable, '-u', '-c', CODE]) - assert output.splitlines() == [ - 'thread started', - 'thread exit', - 'ok!'] diff --git a/pypy/module/thread/test/test_import_lock.py b/pypy/module/thread/test/test_import_lock.py index 989b8759d6..6ca8b71bd7 100644 --- a/pypy/module/thread/test/test_import_lock.py +++ b/pypy/module/thread/test/test_import_lock.py @@ -105,7 +105,7 @@ class TestImportLock: assert importlock.count == 0 # A new module importhook(space, 're') - assert importlock.count >= 9 + assert importlock.count >= 7 # Import it again previous_count = importlock.count importhook(space, 're') diff --git a/pypy/module/time/test/test_time.py b/pypy/module/time/test/test_time.py index 306a804d6c..d8f817977c 100644 --- a/pypy/module/time/test/test_time.py +++ b/pypy/module/time/test/test_time.py @@ -126,10 +126,7 @@ class AppTestTime: assert time.mktime(time.localtime(-1)) == -1 res = time.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - if os.name == 'nt': - assert time.ctime(res) == 'Sat Jan 01 00:00:00 2000' - else: - assert time.ctime(res) == 'Sat Jan 1 00:00:00 2000' + assert time.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py index 98be5a10ea..9efe969a98 100644 --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -40,10 +40,10 @@ class AbstractAttribute(object): assert isinstance(terminator, Terminator) self.terminator = terminator - def read(self, obj, name, index): - attr = self.find_map_attr(name, index) + def read(self, obj, name, attrkind): + attr = self.find_map_attr(name, attrkind) if attr is None: - return self.terminator._read_terminator(obj, name, index) + return self.terminator._read_terminator(obj, name, attrkind) if ( jit.isconstant(attr.storageindex) and jit.isconstant(obj) and @@ -57,20 +57,20 @@ class AbstractAttribute(object): def _pure_mapdict_read_storage(self, obj, storageindex): return obj._mapdict_read_storage(storageindex) - def write(self, obj, name, index, w_value): - attr = self.find_map_attr(name, index) + def write(self, obj, name, attrkind, w_value): + attr = self.find_map_attr(name, attrkind) if attr is None: - return self.terminator._write_terminator(obj, name, index, w_value) + return self.terminator._write_terminator(obj, name, attrkind, w_value) if not attr.ever_mutated: attr.ever_mutated = True obj._mapdict_write_storage(attr.storageindex, w_value) return True - def delete(self, obj, name, index): + def delete(self, obj, name, attrkind): pass @jit.elidable - def find_map_attr(self, name, index): + def find_map_attr(self, name, attrkind): # attr cache space = self.space cache = space.fromcache(MapAttrCache) @@ -84,7 +84,7 @@ class AbstractAttribute(object): c1 = 0x345678 c2 = 1000003 hash_name = objectmodel.compute_hash(name) - hash_selector = intmask((c2 * ((c2 * c1) ^ hash_name)) ^ index) + hash_selector = intmask((c2 * ((c2 * c1) ^ hash_name)) ^ attrkind) product = intmask(attrs_as_int * hash_selector) attr_hash = (r_uint(product) ^ (r_uint(product) << SHIFT1)) >> SHIFT2 # ^^^Note2: same comment too @@ -92,24 +92,24 @@ class AbstractAttribute(object): if cached_attr is self: cached_name = cache.names[attr_hash] cached_index = cache.indexes[attr_hash] - if cached_name == name and cached_index == index: + if cached_name == name and cached_index == attrkind: attr = cache.cached_attrs[attr_hash] if space.config.objspace.std.withmethodcachecounter: cache.hits[name] = cache.hits.get(name, 0) + 1 return attr - attr = self._find_map_attr(name, index) + attr = self._find_map_attr(name, attrkind) if space._side_effects_ok(): cache.attrs[attr_hash] = self cache.names[attr_hash] = name - cache.indexes[attr_hash] = index + cache.indexes[attr_hash] = attrkind cache.cached_attrs[attr_hash] = attr if space.config.objspace.std.withmethodcachecounter: cache.misses[name] = cache.misses.get(name, 0) + 1 return attr - def _find_map_attr(self, name, index): + def _find_map_attr(self, name, attrkind): while isinstance(self, PlainAttribute): - if index == self.index and name == self.name: + if attrkind == self.attrkind and name == self.name: return self self = self.back return None @@ -134,18 +134,18 @@ class AbstractAttribute(object): return None @jit.elidable - def _get_new_attr(self, name, index): + def _get_new_attr(self, name, attrkind): cache = self.cache_attrs if cache is None: cache = self.cache_attrs = {} - attr = cache.get((name, index), None) + attr = cache.get((name, attrkind), None) if attr is None: - attr = PlainAttribute(name, index, self) - cache[name, index] = attr + attr = PlainAttribute(name, attrkind, self) + cache[name, attrkind] = attr return attr - def add_attr(self, obj, name, index, w_value): - self._reorder_and_add(obj, name, index, w_value) + def add_attr(self, obj, name, attrkind, w_value): + self._reorder_and_add(obj, name, attrkind, w_value) if not jit.we_are_jitted(): oldattr = self attr = obj._get_mapdict_map() @@ -154,18 +154,11 @@ class AbstractAttribute(object): assert size_est >= (oldattr.length() * NUM_DIGITS_POW2) oldattr._size_estimate = size_est - def _add_attr_without_reordering(self, obj, name, index, w_value): - attr = self._get_new_attr(name, index) - attr._switch_map_and_write_storage(obj, w_value) - @jit.unroll_safe def _switch_map_and_write_storage(self, obj, w_value): if self.length() > obj._mapdict_storage_length(): - # note that self.size_estimate() is always at least self.length() - new_storage = [None] * self.size_estimate() - for i in range(obj._mapdict_storage_length()): - new_storage[i] = obj._mapdict_read_storage(i) - obj._set_mapdict_storage_and_map(new_storage, self) + obj._set_mapdict_increase_storage(self, w_value) + return # the order is important here: first change the map, then the storage, # for the benefit of the special subclasses @@ -174,13 +167,13 @@ class AbstractAttribute(object): @jit.elidable - def _find_branch_to_move_into(self, name, index): + def _find_branch_to_move_into(self, name, attrkind): # walk up the map chain to find an ancestor with lower order that # already has the current name as a child inserted current_order = sys.maxint number_to_readd = 0 current = self - key = (name, index) + key = (name, attrkind) while True: attr = None if current.cache_attrs is not None: @@ -189,7 +182,7 @@ class AbstractAttribute(object): # we reached the top, so we didn't find it anywhere, # just add it to the top attribute if not isinstance(current, PlainAttribute): - return 0, self._get_new_attr(name, index) + return 0, self._get_new_attr(name, attrkind) else: return number_to_readd, attr @@ -198,11 +191,11 @@ class AbstractAttribute(object): current_order = current.order current = current.back - @jit.look_inside_iff(lambda self, obj, name, index, w_value: + @jit.look_inside_iff(lambda self, obj, name, attrkind, w_value: jit.isconstant(self) and jit.isconstant(name) and - jit.isconstant(index)) - def _reorder_and_add(self, obj, name, index, w_value): + jit.isconstant(attrkind)) + def _reorder_and_add(self, obj, name, attrkind, w_value): # the idea is as follows: the subtrees of any map are ordered by # insertion. the invariant is that subtrees that are inserted later # must not contain the name of the attribute of any earlier inserted @@ -226,7 +219,7 @@ class AbstractAttribute(object): stack_index = 0 while True: current = self - number_to_readd, attr = self._find_branch_to_move_into(name, index) + number_to_readd, attr = self._find_branch_to_move_into(name, attrkind) # we found the attributes further up, need to save the # previous values of the attributes we passed if number_to_readd: @@ -251,7 +244,7 @@ class AbstractAttribute(object): next_map = unerase_map(stack[stack_index]) w_value = unerase_item(stack[stack_index + 1]) name = next_map.name - index = next_map.index + attrkind = next_map.attrkind self = obj._get_mapdict_map() def materialize_r_dict(self, space, obj, dict_w): @@ -274,12 +267,12 @@ class Terminator(AbstractAttribute): AbstractAttribute.__init__(self, space, self) self.w_cls = w_cls - def _read_terminator(self, obj, name, index): + def _read_terminator(self, obj, name, attrkind): return None - def _write_terminator(self, obj, name, index, w_value): - obj._get_mapdict_map().add_attr(obj, name, index, w_value) - if index == DICT and obj._get_mapdict_map().length() >= LIMIT_MAP_ATTRIBUTES: + def _write_terminator(self, obj, name, attrkind, w_value): + obj._get_mapdict_map().add_attr(obj, name, attrkind, w_value) + if attrkind == DICT and obj._get_mapdict_map().length() >= LIMIT_MAP_ATTRIBUTES: space = self.space w_dict = obj.getdict(space) assert isinstance(w_dict, W_DictMultiObject) @@ -329,31 +322,31 @@ class DictTerminator(Terminator): class NoDictTerminator(Terminator): - def _write_terminator(self, obj, name, index, w_value): - if index == DICT: + def _write_terminator(self, obj, name, attrkind, w_value): + if attrkind == DICT: return False - return Terminator._write_terminator(self, obj, name, index, w_value) + return Terminator._write_terminator(self, obj, name, attrkind, w_value) class DevolvedDictTerminator(Terminator): - def _read_terminator(self, obj, name, index): - if index == DICT: + def _read_terminator(self, obj, name, attrkind): + if attrkind == DICT: space = self.space w_dict = obj.getdict(space) return space.finditem_str(w_dict, name) - return Terminator._read_terminator(self, obj, name, index) + return Terminator._read_terminator(self, obj, name, attrkind) - def _write_terminator(self, obj, name, index, w_value): - if index == DICT: + def _write_terminator(self, obj, name, attrkind, w_value): + if attrkind == DICT: space = self.space w_dict = obj.getdict(space) space.setitem_str(w_dict, name, w_value) return True - return Terminator._write_terminator(self, obj, name, index, w_value) + return Terminator._write_terminator(self, obj, name, attrkind, w_value) - def delete(self, obj, name, index): + def delete(self, obj, name, attrkind): from pypy.interpreter.error import OperationError - if index == DICT: + if attrkind == DICT: space = self.space w_dict = obj.getdict(space) try: @@ -362,7 +355,7 @@ class DevolvedDictTerminator(Terminator): if not ex.match(space, space.w_KeyError): raise return Terminator.copy(self, obj) - return Terminator.delete(self, obj, name, index) + return Terminator.delete(self, obj, name, attrkind) def remove_dict_entries(self, obj): assert 0, "should be unreachable" @@ -374,12 +367,12 @@ class DevolvedDictTerminator(Terminator): return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order'] + _immutable_fields_ = ['name', 'attrkind', 'storageindex', 'back', 'ever_mutated?', 'order'] - def __init__(self, name, index, back): + def __init__(self, name, attrkind, back): AbstractAttribute.__init__(self, back.space, back.terminator) self.name = name - self.index = index + self.attrkind = attrkind self.storageindex = back.length() self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 @@ -387,16 +380,16 @@ class PlainAttribute(AbstractAttribute): self.order = len(back.cache_attrs) if back.cache_attrs else 0 def _copy_attr(self, obj, new_obj): - w_value = self.read(obj, self.name, self.index) - new_obj._get_mapdict_map().add_attr(new_obj, self.name, self.index, w_value) + w_value = self.read(obj, self.name, self.attrkind) + new_obj._get_mapdict_map().add_attr(new_obj, self.name, self.attrkind, w_value) - def delete(self, obj, name, index): - if index == self.index and name == self.name: + def delete(self, obj, name, attrkind): + if attrkind == self.attrkind and name == self.name: # ok, attribute is deleted if not self.ever_mutated: self.ever_mutated = True return self.back.copy(obj) - new_obj = self.back.delete(obj, name, index) + new_obj = self.back.delete(obj, name, attrkind) if new_obj is not None: self._copy_attr(obj, new_obj) return new_obj @@ -415,13 +408,13 @@ class PlainAttribute(AbstractAttribute): return new_obj def search(self, attrtype): - if self.index == attrtype: + if self.attrkind == attrtype: return self return self.back.search(attrtype) def materialize_r_dict(self, space, obj, dict_w): new_obj = self.back.materialize_r_dict(space, obj, dict_w) - if self.index == DICT: + if self.attrkind == DICT: w_attr = space.newtext(self.name) dict_w[w_attr] = obj._mapdict_read_storage(self.storageindex) else: @@ -430,7 +423,7 @@ class PlainAttribute(AbstractAttribute): def materialize_str_dict(self, space, obj, str_dict): new_obj = self.back.materialize_str_dict(space, obj, str_dict) - if self.index == DICT: + if self.attrkind == DICT: str_dict[self.name] = obj._mapdict_read_storage(self.storageindex) else: self._copy_attr(obj, new_obj) @@ -438,12 +431,12 @@ class PlainAttribute(AbstractAttribute): def remove_dict_entries(self, obj): new_obj = self.back.remove_dict_entries(obj) - if self.index != DICT: + if self.attrkind != DICT: self._copy_attr(obj, new_obj) return new_obj def __repr__(self): - return "<PlainAttribute %s %s %s %r>" % (self.name, self.index, self.storageindex, self.back) + return "<PlainAttribute %s %s %s %r>" % (self.name, self.attrkind, self.storageindex, self.back) class MapAttrCache(object): def __init__(self, space): @@ -511,16 +504,16 @@ class BaseUserClassMapdict: # methods needed for slots def getslotvalue(self, slotindex): - index = SLOTS_STARTING_FROM + slotindex - return self._get_mapdict_map().read(self, "slot", index) + attrkind = SLOTS_STARTING_FROM + slotindex + return self._get_mapdict_map().read(self, "slot", attrkind) def setslotvalue(self, slotindex, w_value): - index = SLOTS_STARTING_FROM + slotindex - self._get_mapdict_map().write(self, "slot", index, w_value) + attrkind = SLOTS_STARTING_FROM + slotindex + self._get_mapdict_map().write(self, "slot", attrkind, w_value) def delslotvalue(self, slotindex): - index = SLOTS_STARTING_FROM + slotindex - new_obj = self._get_mapdict_map().delete(self, "slot", index) + attrkind = SLOTS_STARTING_FROM + slotindex + new_obj = self._get_mapdict_map().delete(self, "slot", attrkind) if new_obj is None: return False self._set_mapdict_storage_and_map(new_obj.storage, new_obj.map) @@ -631,9 +624,20 @@ class MapdictStorageMixin(object): self.storage[storageindex] = value def _mapdict_storage_length(self): + """ return the size of the storage (which should be longer or equal in + size to self.map.length() due to overallocation). """ return len(self.storage) + def _set_mapdict_increase_storage(self, map, w_value): + """ increase storage size, adding w_value """ + len_storage = len(self.storage) + new_storage = self.storage + [None] * (map.size_estimate() - len_storage) + new_storage[len_storage] = w_value + self.map = map + self.storage = new_storage + def _set_mapdict_storage_and_map(self, storage, map): + """ store a new complete storage list, and also a new map """ self.storage = storage self.map = map @@ -736,6 +740,23 @@ def _make_storage_mixin_size_n(n=SUBCLASSES_NUM_FIELDS): erased = erase_list(storage_list) setattr(self, "_value%s" % nmin1, erased) + def _set_mapdict_increase_storage(self, map, w_value): + len_storage = self.map.length() + if len_storage <= nmin1: + self._mapdict_write_storage(len_storage, w_value) + self.map = map + return + if len_storage == n: + erased = getattr(self, "_value%s" % nmin1) + new_storage = [unerase_item(erased), w_value] + else: + new_storage = [None] * (map.size_estimate() - self._mapdict_storage_length()) + new_storage = self._mapdict_get_storage_list() + new_storage + new_storage[len_storage - nmin1] = w_value + self.map = map + erased = erase_list(new_storage) + setattr(self, "_value%s" % nmin1, erased) + subcls.__name__ = "Size%s" % n return subcls @@ -1043,9 +1064,9 @@ def LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map): _, w_descr = w_type._pure_lookup_where_with_method_cache( name, version_tag) # - attrname, index = ("", INVALID) + attrname, attrkind = ("", INVALID) if w_descr is None: - attrname, index = (name, DICT) # common case: no such attr in the class + attrname, attrkind = (name, DICT) # common case: no such attr in the class elif isinstance(w_descr, MutableCell): pass # we have a MutableCell in the class: give up elif space.is_data_descr(w_descr): @@ -1053,7 +1074,7 @@ def LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map): # (if any) has no relevance. from pypy.interpreter.typedef import Member if isinstance(w_descr, Member): # it is a slot -- easy case - attrname, index = ("slot", SLOTS_STARTING_FROM + w_descr.index) + attrname, attrkind = ("slot", SLOTS_STARTING_FROM + w_descr.index) else: # There is a non-data descriptor in the class. If there is # also a dict attribute, use the latter, caching its storageindex. @@ -1061,14 +1082,14 @@ def LOAD_ATTR_slowpath(pycode, w_obj, nameindex, map): # but we don't care too much; the common case of a method # invocation is handled by LOOKUP_METHOD_xxx below. attrname = name - index = DICT + attrkind = DICT # - if index != INVALID: - attr = map.find_map_attr(attrname, index) + if attrkind != INVALID: + attr = map.find_map_attr(attrname, attrkind) if attr is not None: # Note that if map.terminator is a DevolvedDictTerminator # or the class provides its own dict, not using mapdict, then: - # map.find_map_attr will always return None if index==DICT. + # map.find_map_attr will always return None if attrkind==DICT. _fill_cache(pycode, nameindex, map, version_tag, attr.storageindex) return w_obj._mapdict_read_storage(attr.storageindex) if space.config.objspace.std.withmethodcachecounter: diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py index c81328fc79..6fb461a734 100644 --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -496,12 +496,16 @@ class W_BaseSetObject(W_Root): @gateway.unwrap_spec(others_w='args_w') def descr_update(self, space, others_w): """Update a set with the union of itself and another.""" + self._descr_update(space, others_w) + + @jit.look_inside_iff(lambda self, space, others_w: + jit.loop_unrolling_heuristic(others_w, len(others_w), UNROLL_CUTOFF)) + def _descr_update(self, space, others_w): for w_other in others_w: if isinstance(w_other, W_BaseSetObject): self.update(w_other) else: - for w_key in space.listview(w_other): - self.add(w_key) + _update_from_iterable(space, self, w_other) class W_SetObject(W_BaseSetObject): @@ -1636,10 +1640,12 @@ def set_strategy_and_setdata(space, w_set, w_iterable): length_hint = space.length_hint(w_iterable, 0) - if jit.isconstant(length_hint): + if jit.isconstant(length_hint) and length_hint: return _pick_correct_strategy_unroll(space, w_set, w_iterable) - _create_from_iterable(space, w_set, w_iterable) + w_set.strategy = strategy = space.fromcache(EmptySetStrategy) + w_set.sstorage = strategy.get_empty_storage() + _update_from_iterable(space, w_set, w_iterable) @jit.unroll_safe @@ -1687,17 +1693,14 @@ def _pick_correct_strategy_unroll(space, w_set, w_iterable): def get_printable_location(tp, strategy): - return "create_set: %s %s" % (tp.iterator_greenkey_printable(), strategy) + return "update_set: %s %s" % (tp.iterator_greenkey_printable(), strategy) -create_set_driver = jit.JitDriver(name='create_set', +update_set_driver = jit.JitDriver(name='update_set', greens=['tp', 'strategy'], reds='auto', get_printable_location=get_printable_location) -def _create_from_iterable(space, w_set, w_iterable): - w_set.strategy = strategy = space.fromcache(EmptySetStrategy) - w_set.sstorage = strategy.get_empty_storage() - +def _update_from_iterable(space, w_set, w_iterable): tp = space.iterator_greenkey(w_iterable) w_iter = space.iter(w_iterable) @@ -1708,7 +1711,7 @@ def _create_from_iterable(space, w_set, w_iterable): if not e.match(space, space.w_StopIteration): raise return - create_set_driver.jit_merge_point(tp=tp, strategy=w_set.strategy) + update_set_driver.jit_merge_point(tp=tp, strategy=w_set.strategy) w_set.add(w_item) diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py index 3d0e1cc477..a68ffe24f5 100644 --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -433,6 +433,11 @@ class AppTestAppSetTest: s1 = set([1, 2.0, "3"]) s1.update(set(["3", 4, 5.0])) + def test_update_not_iterable_error(self): + with raises(TypeError) as e: + set().update(1) + assert "'int' object is not iterable" in str(e.value) + def test_recursive_repr(self): class A(object): def __init__(self, s): diff --git a/pypy/tool/release/check_versions.py b/pypy/tool/release/check_versions.py new file mode 100644 index 0000000000..3a8e7c7fbf --- /dev/null +++ b/pypy/tool/release/check_versions.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 + +""" +Verify the versions.json file that describes the valid downloads. +- all the files should be valid +- the other fields should have valid values +- the pypy_version should be in the repo tags + +Can be run as check_versions.py <filename>, by default will +download https://buildbot.pypy.org/pypy/versions.json and parse it + +""" + +import json +from urllib import request, error +import sys + + +def assert_equal(a, b): + if a != b: + raise ValueError(f"'{a}' != '{b}'") + + +def assert_different(a, b): + if a == b: + raise ValueError(f"'{a}' == '{b}'") + + +def assert_in(a, b): + if a not in b: + raise ValueError(f"'{a}' not in '{b}'") + + + + +pypy_versions = {'7.3.3': {'python_version': ['3.7.9', '3.6.12', '2.7.18'], + 'date': '2020-11-21', + }, + '7.3.3rc1': {'python_version': ['3.6.12'], + 'date': '2020-11-11', + }, + '7.3.2': {'python_version': ['3.7.9', '3.6.9', '2.7.13'], + 'date': '2020-09-25', + }, + 'nightly': {'python_version': ['2.7', '3.6', '3.7']}, + } + + +def create_latest_versions(v): + """Create a dictionary with key of cpython_version and value of the latest + pypy version for that cpython""" + ret = {} + for pypy_ver, vv in v.items(): + for pv in vv['python_version']: + # for nightlies, we rely on python_version being major.minor while + # for releases python_version is major.minor.patch + if pv not in ret or ( + vv['date'] > v[ret[pv]]['date']): + ret[pv] = pypy_ver + return ret + + +latest_pypys = create_latest_versions(pypy_versions) + +arches = ['aarch64', 'i686', 'x64', 'x86', 'darwin', 's390x'] +platforms = ['linux', 'win32', 'darwin'] +arch_map={('aarch64', 'linux'): 'aarch64', + ('i686', 'linux'): 'linux32', + ('x64', 'linux'): 'linux64', + ('s390x', 'linux'): 's390x', + ('x86', 'win32'): 'win32', + ('x64', 'darwin'): 'osx64', + } + + +def check_versions(data): + for d in data: + assert_in(d['pypy_version'], pypy_versions) + v = pypy_versions[d['pypy_version']] + assert_in(d['python_version'], v['python_version']) + if ('rc' in d['pypy_version'] or 'nightly' in d['pypy_version']): + assert d['stable'] is False + else: + assert d['stable'] is True + if d['pypy_version'] == 'nightly': + assert d['latest_pypy'] is False + elif d['latest_pypy'] is True: + assert_equal(latest_pypys[d['python_version']], d['pypy_version']) + else: + assert_different(latest_pypys[d['python_version']], d['pypy_version']) + if 'date' in d: + assert_equal(d['date'], v['date']) + for f in d['files']: + if 'rc' not in d['pypy_version']: + assert_in(f['filename'], f['download_url']) + assert_in(d['pypy_version'], f['download_url']) + assert_in(f['arch'], arches) + assert_in(f['platform'], platforms) + arch_plat = arch_map[(f['arch'], f['platform'])] + if d['pypy_version'] == 'nightly': + if arch_plat == 'linux32': + # the nightly builds have a quirk in the linux32 file name + arch_plat = 'linux' + assert_in(arch_plat, f['download_url']) + else: + assert_in(arch_plat, f['download_url']) + assert_in('.'.join(d['python_version'].split('.')[:2]), f['download_url']) + try: + r = request.urlopen(f['download_url']) + except error.HTTPError as e: + raise ValueError(f"could not open {f['download_url']}") from None + assert_equal(r.getcode(), 200) + +if __name__ == '__main__': + if len(sys.argv) > 1: + print(f'checking local file "{sys.argv[1]}"') + with open(sys.argv[1]) as fid: + data = json.loads(fid.read()) + else: + print('downloading versions.json') + response = request.urlopen('https://buildbot.pypy.org/pypy/versions.json') + assert_equal(response.getcode(), 200) + data = json.loads(response.read()) + check_versions(data) diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh index b6b4f801c2..29636ab220 100755 --- a/pypy/tool/release/repackage.sh +++ b/pypy/tool/release/repackage.sh @@ -1,12 +1,12 @@ #! /bin/bash # Edit these appropriately before running this script -pmaj=2 # python main version: 2 or 3 -pmin=7 # python minor version +pmaj=3 # python main version: 2 or 3 +pmin=6 # python minor version maj=7 min=3 -rev=2 -# rc=rc3 # set to blank for actual release +rev=3 +#rc=rc2 # set to blank for actual release function maybe_exit { if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then diff --git a/pypy/tool/release/versions.json b/pypy/tool/release/versions.json new file mode 100644 index 0000000000..b6ea93a2d4 --- /dev/null +++ b/pypy/tool/release/versions.json @@ -0,0 +1,449 @@ +[ + { + "pypy_version": "7.3.3", + "python_version": "3.6.12", + "stable": true, + "latest_pypy": true, + "date": "2020-11-21", + "files": [ + { + "filename": "pypy3.6-v7.3.3-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-aarch64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux32.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-win32.zip" + }, + { + "filename": "pypy3.6-v7.3.3-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.3rc1", + "python_version": "3.6.12", + "stable": false, + "latest_pypy": false, + "date": "2020-11-11", + "files": [ + { + "filename": "pypy3.6-v7.3.3rc1-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-aarch64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-linux32rc1.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-linux32.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3rc1-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-linux64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3rc1-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-osx64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.3-win32rc1.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-win32.zip" + }, + { + "filename": "pypy3.6-v7.3.3rc1-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3rc1-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.3", + "python_version": "3.7.9", + "stable": true, + "latest_pypy": true, + "date": "2020-11-21", + "files": [ + { + "filename": "pypy3.7-v7.3.3-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-aarch64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.3-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux32.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.3-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.3-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.3-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-win32.zip" + }, + { + "filename": "pypy3.7-v7.3.3-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.3", + "python_version": "2.7.18", + "stable": true, + "latest_pypy": true, + "date": "2020-11-21", + "files": [ + { + "filename": "pypy2.7-v7.3.3-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-aarch64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.3-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux32.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.3-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.3-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.3-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-win32.zip" + }, + { + "filename": "pypy2.7-v7.3.3-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.2", + "python_version": "3.6.9", + "stable": true, + "latest_pypy": true, + "date": "2020-09-25", + "files": [ + { + "filename": "pypy3.6-v7.3.2-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-aarch64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.2-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-linux32.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.2-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-linux64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.2-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-osx64.tar.bz2" + }, + { + "filename": "pypy3.6-v7.3.2-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-win32.zip" + }, + { + "filename": "pypy3.6-v7.3.2-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.6-v7.3.2-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.2", + "python_version": "3.7.9", + "stable": true, + "latest_pypy": false, + "date": "2020-09-25", + "files": [ + { + "filename": "pypy3.7-v7.3.2-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-aarch64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.2-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-linux32.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.2-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-linux64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.2-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-osx64.tar.bz2" + }, + { + "filename": "pypy3.7-v7.3.2-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-win32.zip" + }, + { + "filename": "pypy3.7-v7.3.2-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy3.7-v7.3.2-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "7.3.2", + "python_version": "2.7.13", + "stable": true, + "latest_pypy": true, + "date": "2020-09-25", + "files": [ + { + "filename": "pypy2.7-v7.3.2-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-aarch64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.2-linux32.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-linux32.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.2-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-linux64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.2-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-osx64.tar.bz2" + }, + { + "filename": "pypy2.7-v7.3.2-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-win32.zip" + }, + { + "filename": "pypy2.7-v7.3.2-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "https://downloads.python.org/pypy/pypy2.7-v7.3.2-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "nightly", + "python_version": "2.7", + "stable": false, + "latest_pypy": false, + "files": [ + { + "filename": "pypy-c-jit-latest-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-aarch64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-linux.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-linux64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-osx64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-win32.zip" + }, + { + "filename": "pypy-c-jit-latest-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/trunk/pypy-c-jit-latest-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "nightly", + "python_version": "3.7", + "stable": false, + "latest_pypy": false, + "files": [ + { + "filename": "pypy-c-jit-latest-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-aarch64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-linux.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-linux64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-osx64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-win32.zip" + }, + { + "filename": "pypy-c-jit-latest-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-s390x.tar.bz2" + } + ] + }, + { + "pypy_version": "nightly", + "python_version": "3.6", + "stable": false, + "latest_pypy": false, + "files": [ + { + "filename": "pypy-c-jit-latest-aarch64.tar.bz2", + "arch": "aarch64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-aarch64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux.tar.bz2", + "arch": "i686", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-linux64.tar.bz2", + "arch": "x64", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-osx64.tar.bz2", + "arch": "x64", + "platform": "darwin", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-osx64.tar.bz2" + }, + { + "filename": "pypy-c-jit-latest-win32.zip", + "arch": "x86", + "platform": "win32", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-win32.zip" + }, + { + "filename": "pypy-c-jit-latest-s390x.tar.bz2", + "arch": "s390x", + "platform": "linux", + "download_url": "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-s390x.tar.bz2" + } + ] + } +] diff --git a/pypy/tool/test/test_tab.py b/pypy/tool/test/test_tab.py index 0aea831a39..ce96b61cf0 100644 --- a/pypy/tool/test/test_tab.py +++ b/pypy/tool/test/test_tab.py @@ -44,6 +44,8 @@ def test_no_pypy_import_in_rpython(): if os.path.isfile(path): if not path.lower().endswith('.py'): return + if path.lower().endswith('rsre_constants.py'): + return # exception in this file with file(path) as f: for line in f: if "import" not in line: |