diff options
| author |   Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> | 2021-05-17 10:34:39 -0700 |
|---|---|---|
| committer |   Michał Górny <mgorny@gentoo.org> | 2022-12-07 08:54:58 +0100 |
| commit | a62b59f53f4c44baf93668507811da766c79a8f6 (patch) | |
| tree | 6b793b99e5f07eb9920e2949ae8dc5e8514e4112 | |
| parent | ssl: Hard-disable SSLv3 to avoid automagic deps (diff) | |
| download | cpython-a62b59f53f4c44baf93668507811da766c79a8f6.tar.gz cpython-a62b59f53f4c44baf93668507811da766c79a8f6.tar.bz2 cpython-a62b59f53f4c44baf93668507811da766c79a8f6.zip | |
bpo-43650: Fix MemoryError on zip.read in shutil._unpack_zipfile for large files (GH-25058)
`shutil.unpack_archive()` tries to read the whole file into memory, making no use of any kind of smaller buffer. Process crashes for really large files: I.e. archive: ~1.7G, unpacked: ~10G. Before the crash it can easily take away all available RAM on smaller systems. Had to pull the code form `zipfile.Zipfile.extractall()` to fix this
Automerge-Triggered-By: GH:gpshead
(cherry picked from commit f32c7950e0077b6d9a8e217c2796fc582f18ca08)
Co-authored-by: Igor Bolshakov <ibolsch@gmail.com>
| -rw-r--r-- | Lib/shutil.py | 16 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst | 2 |
2 files changed, 8 insertions, 10 deletions
diff --git a/Lib/shutil.py b/Lib/shutil.py index fdadb838000..aaf76c651ba 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -1144,20 +1144,16 @@ def _unpack_zipfile(filename, extract_dir): if name.startswith('/') or '..' in name: continue - target = os.path.join(extract_dir, *name.split('/')) - if not target: + targetpath = os.path.join(extract_dir, *name.split('/')) + if not targetpath: continue - _ensure_directory(target) + _ensure_directory(targetpath) if not name.endswith('/'): # file - data = zip.read(info.filename) - f = open(target, 'wb') - try: - f.write(data) - finally: - f.close() - del data + with zip.open(name, 'r') as source, \ + open(targetpath, 'wb') as target: + copyfileobj(source, target) finally: zip.close() diff --git a/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst b/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst new file mode 100644 index 00000000000..a2ea4a4800a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-29-00-23-30.bpo-43650.v01tic.rst @@ -0,0 +1,2 @@ +Fix :exc:`MemoryError` in :func:`shutil.unpack_archive` which fails inside +:func:`shutil._unpack_zipfile` on large files. Patch by Igor Bolshakov. |