_tempfile.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613
  1. from __future__ import annotations
  2. import os
  3. import sys
  4. import tempfile
  5. from collections.abc import Iterable
  6. from io import BytesIO, TextIOWrapper
  7. from types import TracebackType
  8. from typing import (
  9. TYPE_CHECKING,
  10. Any,
  11. AnyStr,
  12. Generic,
  13. overload,
  14. )
  15. from .. import to_thread
  16. from .._core._fileio import AsyncFile
  17. from ..lowlevel import checkpoint_if_cancelled
  18. if TYPE_CHECKING:
  19. from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer
  20. class TemporaryFile(Generic[AnyStr]):
  21. """
  22. An asynchronous temporary file that is automatically created and cleaned up.
  23. This class provides an asynchronous context manager interface to a temporary file.
  24. The file is created using Python's standard `tempfile.TemporaryFile` function in a
  25. background thread, and is wrapped as an asynchronous file using `AsyncFile`.
  26. :param mode: The mode in which the file is opened. Defaults to "w+b".
  27. :param buffering: The buffering policy (-1 means the default buffering).
  28. :param encoding: The encoding used to decode or encode the file. Only applicable in
  29. text mode.
  30. :param newline: Controls how universal newlines mode works (only applicable in text
  31. mode).
  32. :param suffix: The suffix for the temporary file name.
  33. :param prefix: The prefix for the temporary file name.
  34. :param dir: The directory in which the temporary file is created.
  35. :param errors: The error handling scheme used for encoding/decoding errors.
  36. """
  37. _async_file: AsyncFile[AnyStr]
  38. @overload
  39. def __init__(
  40. self: TemporaryFile[bytes],
  41. mode: OpenBinaryMode = ...,
  42. buffering: int = ...,
  43. encoding: str | None = ...,
  44. newline: str | None = ...,
  45. suffix: str | None = ...,
  46. prefix: str | None = ...,
  47. dir: str | None = ...,
  48. *,
  49. errors: str | None = ...,
  50. ): ...
  51. @overload
  52. def __init__(
  53. self: TemporaryFile[str],
  54. mode: OpenTextMode,
  55. buffering: int = ...,
  56. encoding: str | None = ...,
  57. newline: str | None = ...,
  58. suffix: str | None = ...,
  59. prefix: str | None = ...,
  60. dir: str | None = ...,
  61. *,
  62. errors: str | None = ...,
  63. ): ...
  64. def __init__(
  65. self,
  66. mode: OpenTextMode | OpenBinaryMode = "w+b",
  67. buffering: int = -1,
  68. encoding: str | None = None,
  69. newline: str | None = None,
  70. suffix: str | None = None,
  71. prefix: str | None = None,
  72. dir: str | None = None,
  73. *,
  74. errors: str | None = None,
  75. ) -> None:
  76. self.mode = mode
  77. self.buffering = buffering
  78. self.encoding = encoding
  79. self.newline = newline
  80. self.suffix: str | None = suffix
  81. self.prefix: str | None = prefix
  82. self.dir: str | None = dir
  83. self.errors = errors
  84. async def __aenter__(self) -> AsyncFile[AnyStr]:
  85. fp = await to_thread.run_sync(
  86. lambda: tempfile.TemporaryFile(
  87. self.mode,
  88. self.buffering,
  89. self.encoding,
  90. self.newline,
  91. self.suffix,
  92. self.prefix,
  93. self.dir,
  94. errors=self.errors,
  95. )
  96. )
  97. self._async_file = AsyncFile(fp)
  98. return self._async_file
  99. async def __aexit__(
  100. self,
  101. exc_type: type[BaseException] | None,
  102. exc_value: BaseException | None,
  103. traceback: TracebackType | None,
  104. ) -> None:
  105. await self._async_file.aclose()
  106. class NamedTemporaryFile(Generic[AnyStr]):
  107. """
  108. An asynchronous named temporary file that is automatically created and cleaned up.
  109. This class provides an asynchronous context manager for a temporary file with a
  110. visible name in the file system. It uses Python's standard
  111. :func:`~tempfile.NamedTemporaryFile` function and wraps the file object with
  112. :class:`AsyncFile` for asynchronous operations.
  113. :param mode: The mode in which the file is opened. Defaults to "w+b".
  114. :param buffering: The buffering policy (-1 means the default buffering).
  115. :param encoding: The encoding used to decode or encode the file. Only applicable in
  116. text mode.
  117. :param newline: Controls how universal newlines mode works (only applicable in text
  118. mode).
  119. :param suffix: The suffix for the temporary file name.
  120. :param prefix: The prefix for the temporary file name.
  121. :param dir: The directory in which the temporary file is created.
  122. :param delete: Whether to delete the file when it is closed.
  123. :param errors: The error handling scheme used for encoding/decoding errors.
  124. :param delete_on_close: (Python 3.12+) Whether to delete the file on close.
  125. """
  126. _async_file: AsyncFile[AnyStr]
  127. @overload
  128. def __init__(
  129. self: NamedTemporaryFile[bytes],
  130. mode: OpenBinaryMode = ...,
  131. buffering: int = ...,
  132. encoding: str | None = ...,
  133. newline: str | None = ...,
  134. suffix: str | None = ...,
  135. prefix: str | None = ...,
  136. dir: str | None = ...,
  137. delete: bool = ...,
  138. *,
  139. errors: str | None = ...,
  140. delete_on_close: bool = ...,
  141. ): ...
  142. @overload
  143. def __init__(
  144. self: NamedTemporaryFile[str],
  145. mode: OpenTextMode,
  146. buffering: int = ...,
  147. encoding: str | None = ...,
  148. newline: str | None = ...,
  149. suffix: str | None = ...,
  150. prefix: str | None = ...,
  151. dir: str | None = ...,
  152. delete: bool = ...,
  153. *,
  154. errors: str | None = ...,
  155. delete_on_close: bool = ...,
  156. ): ...
  157. def __init__(
  158. self,
  159. mode: OpenBinaryMode | OpenTextMode = "w+b",
  160. buffering: int = -1,
  161. encoding: str | None = None,
  162. newline: str | None = None,
  163. suffix: str | None = None,
  164. prefix: str | None = None,
  165. dir: str | None = None,
  166. delete: bool = True,
  167. *,
  168. errors: str | None = None,
  169. delete_on_close: bool = True,
  170. ) -> None:
  171. self._params: dict[str, Any] = {
  172. "mode": mode,
  173. "buffering": buffering,
  174. "encoding": encoding,
  175. "newline": newline,
  176. "suffix": suffix,
  177. "prefix": prefix,
  178. "dir": dir,
  179. "delete": delete,
  180. "errors": errors,
  181. }
  182. if sys.version_info >= (3, 12):
  183. self._params["delete_on_close"] = delete_on_close
  184. async def __aenter__(self) -> AsyncFile[AnyStr]:
  185. fp = await to_thread.run_sync(
  186. lambda: tempfile.NamedTemporaryFile(**self._params)
  187. )
  188. self._async_file = AsyncFile(fp)
  189. return self._async_file
  190. async def __aexit__(
  191. self,
  192. exc_type: type[BaseException] | None,
  193. exc_value: BaseException | None,
  194. traceback: TracebackType | None,
  195. ) -> None:
  196. await self._async_file.aclose()
  197. class SpooledTemporaryFile(AsyncFile[AnyStr]):
  198. """
  199. An asynchronous spooled temporary file that starts in memory and is spooled to disk.
  200. This class provides an asynchronous interface to a spooled temporary file, much like
  201. Python's standard :class:`~tempfile.SpooledTemporaryFile`. It supports asynchronous
  202. write operations and provides a method to force a rollover to disk.
  203. :param max_size: Maximum size in bytes before the file is rolled over to disk.
  204. :param mode: The mode in which the file is opened. Defaults to "w+b".
  205. :param buffering: The buffering policy (-1 means the default buffering).
  206. :param encoding: The encoding used to decode or encode the file (text mode only).
  207. :param newline: Controls how universal newlines mode works (text mode only).
  208. :param suffix: The suffix for the temporary file name.
  209. :param prefix: The prefix for the temporary file name.
  210. :param dir: The directory in which the temporary file is created.
  211. :param errors: The error handling scheme used for encoding/decoding errors.
  212. """
  213. _rolled: bool = False
  214. @overload
  215. def __init__(
  216. self: SpooledTemporaryFile[bytes],
  217. max_size: int = ...,
  218. mode: OpenBinaryMode = ...,
  219. buffering: int = ...,
  220. encoding: str | None = ...,
  221. newline: str | None = ...,
  222. suffix: str | None = ...,
  223. prefix: str | None = ...,
  224. dir: str | None = ...,
  225. *,
  226. errors: str | None = ...,
  227. ): ...
  228. @overload
  229. def __init__(
  230. self: SpooledTemporaryFile[str],
  231. max_size: int = ...,
  232. mode: OpenTextMode = ...,
  233. buffering: int = ...,
  234. encoding: str | None = ...,
  235. newline: str | None = ...,
  236. suffix: str | None = ...,
  237. prefix: str | None = ...,
  238. dir: str | None = ...,
  239. *,
  240. errors: str | None = ...,
  241. ): ...
  242. def __init__(
  243. self,
  244. max_size: int = 0,
  245. mode: OpenBinaryMode | OpenTextMode = "w+b",
  246. buffering: int = -1,
  247. encoding: str | None = None,
  248. newline: str | None = None,
  249. suffix: str | None = None,
  250. prefix: str | None = None,
  251. dir: str | None = None,
  252. *,
  253. errors: str | None = None,
  254. ) -> None:
  255. self._tempfile_params: dict[str, Any] = {
  256. "mode": mode,
  257. "buffering": buffering,
  258. "encoding": encoding,
  259. "newline": newline,
  260. "suffix": suffix,
  261. "prefix": prefix,
  262. "dir": dir,
  263. "errors": errors,
  264. }
  265. self._max_size = max_size
  266. if "b" in mode:
  267. super().__init__(BytesIO()) # type: ignore[arg-type]
  268. else:
  269. super().__init__(
  270. TextIOWrapper( # type: ignore[arg-type]
  271. BytesIO(),
  272. encoding=encoding,
  273. errors=errors,
  274. newline=newline,
  275. write_through=True,
  276. )
  277. )
  278. async def aclose(self) -> None:
  279. if not self._rolled:
  280. self._fp.close()
  281. return
  282. await super().aclose()
  283. async def _check(self) -> None:
  284. if self._rolled or self._fp.tell() <= self._max_size:
  285. return
  286. await self.rollover()
  287. async def rollover(self) -> None:
  288. if self._rolled:
  289. return
  290. self._rolled = True
  291. buffer = self._fp
  292. buffer.seek(0)
  293. self._fp = await to_thread.run_sync(
  294. lambda: tempfile.TemporaryFile(**self._tempfile_params)
  295. )
  296. await self.write(buffer.read())
  297. buffer.close()
  298. @property
  299. def closed(self) -> bool:
  300. return self._fp.closed
  301. async def read(self, size: int = -1) -> AnyStr:
  302. if not self._rolled:
  303. await checkpoint_if_cancelled()
  304. return self._fp.read(size)
  305. return await super().read(size) # type: ignore[return-value]
  306. async def read1(self: SpooledTemporaryFile[bytes], size: int = -1) -> bytes:
  307. if not self._rolled:
  308. await checkpoint_if_cancelled()
  309. return self._fp.read1(size)
  310. return await super().read1(size)
  311. async def readline(self) -> AnyStr:
  312. if not self._rolled:
  313. await checkpoint_if_cancelled()
  314. return self._fp.readline()
  315. return await super().readline() # type: ignore[return-value]
  316. async def readlines(self) -> list[AnyStr]:
  317. if not self._rolled:
  318. await checkpoint_if_cancelled()
  319. return self._fp.readlines()
  320. return await super().readlines() # type: ignore[return-value]
  321. async def readinto(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int:
  322. if not self._rolled:
  323. await checkpoint_if_cancelled()
  324. self._fp.readinto(b)
  325. return await super().readinto(b)
  326. async def readinto1(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int:
  327. if not self._rolled:
  328. await checkpoint_if_cancelled()
  329. self._fp.readinto(b)
  330. return await super().readinto1(b)
  331. async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int:
  332. if not self._rolled:
  333. await checkpoint_if_cancelled()
  334. return self._fp.seek(offset, whence)
  335. return await super().seek(offset, whence)
  336. async def tell(self) -> int:
  337. if not self._rolled:
  338. await checkpoint_if_cancelled()
  339. return self._fp.tell()
  340. return await super().tell()
  341. async def truncate(self, size: int | None = None) -> int:
  342. if not self._rolled:
  343. await checkpoint_if_cancelled()
  344. return self._fp.truncate(size)
  345. return await super().truncate(size)
  346. @overload
  347. async def write(self: SpooledTemporaryFile[bytes], b: ReadableBuffer) -> int: ...
  348. @overload
  349. async def write(self: SpooledTemporaryFile[str], b: str) -> int: ...
  350. async def write(self, b: ReadableBuffer | str) -> int:
  351. """
  352. Asynchronously write data to the spooled temporary file.
  353. If the file has not yet been rolled over, the data is written synchronously,
  354. and a rollover is triggered if the size exceeds the maximum size.
  355. :param s: The data to write.
  356. :return: The number of bytes written.
  357. :raises RuntimeError: If the underlying file is not initialized.
  358. """
  359. if not self._rolled:
  360. await checkpoint_if_cancelled()
  361. result = self._fp.write(b)
  362. await self._check()
  363. return result
  364. return await super().write(b) # type: ignore[misc]
  365. @overload
  366. async def writelines(
  367. self: SpooledTemporaryFile[bytes], lines: Iterable[ReadableBuffer]
  368. ) -> None: ...
  369. @overload
  370. async def writelines(
  371. self: SpooledTemporaryFile[str], lines: Iterable[str]
  372. ) -> None: ...
  373. async def writelines(self, lines: Iterable[str] | Iterable[ReadableBuffer]) -> None:
  374. """
  375. Asynchronously write a list of lines to the spooled temporary file.
  376. If the file has not yet been rolled over, the lines are written synchronously,
  377. and a rollover is triggered if the size exceeds the maximum size.
  378. :param lines: An iterable of lines to write.
  379. :raises RuntimeError: If the underlying file is not initialized.
  380. """
  381. if not self._rolled:
  382. await checkpoint_if_cancelled()
  383. result = self._fp.writelines(lines)
  384. await self._check()
  385. return result
  386. return await super().writelines(lines) # type: ignore[misc]
  387. class TemporaryDirectory(Generic[AnyStr]):
  388. """
  389. An asynchronous temporary directory that is created and cleaned up automatically.
  390. This class provides an asynchronous context manager for creating a temporary
  391. directory. It wraps Python's standard :class:`~tempfile.TemporaryDirectory` to
  392. perform directory creation and cleanup operations in a background thread.
  393. :param suffix: Suffix to be added to the temporary directory name.
  394. :param prefix: Prefix to be added to the temporary directory name.
  395. :param dir: The parent directory where the temporary directory is created.
  396. :param ignore_cleanup_errors: Whether to ignore errors during cleanup
  397. :param delete: Whether to delete the directory upon closing (Python 3.12+).
  398. """
  399. def __init__(
  400. self,
  401. suffix: AnyStr | None = None,
  402. prefix: AnyStr | None = None,
  403. dir: AnyStr | None = None,
  404. *,
  405. ignore_cleanup_errors: bool = False,
  406. delete: bool = True,
  407. ) -> None:
  408. self.suffix: AnyStr | None = suffix
  409. self.prefix: AnyStr | None = prefix
  410. self.dir: AnyStr | None = dir
  411. self.ignore_cleanup_errors = ignore_cleanup_errors
  412. self.delete = delete
  413. self._tempdir: tempfile.TemporaryDirectory | None = None
  414. async def __aenter__(self) -> str:
  415. params: dict[str, Any] = {
  416. "suffix": self.suffix,
  417. "prefix": self.prefix,
  418. "dir": self.dir,
  419. "ignore_cleanup_errors": self.ignore_cleanup_errors,
  420. }
  421. if sys.version_info >= (3, 12):
  422. params["delete"] = self.delete
  423. self._tempdir = await to_thread.run_sync(
  424. lambda: tempfile.TemporaryDirectory(**params)
  425. )
  426. return await to_thread.run_sync(self._tempdir.__enter__)
  427. async def __aexit__(
  428. self,
  429. exc_type: type[BaseException] | None,
  430. exc_value: BaseException | None,
  431. traceback: TracebackType | None,
  432. ) -> None:
  433. if self._tempdir is not None:
  434. await to_thread.run_sync(
  435. self._tempdir.__exit__, exc_type, exc_value, traceback
  436. )
  437. async def cleanup(self) -> None:
  438. if self._tempdir is not None:
  439. await to_thread.run_sync(self._tempdir.cleanup)
  440. @overload
  441. async def mkstemp(
  442. suffix: str | None = None,
  443. prefix: str | None = None,
  444. dir: str | None = None,
  445. text: bool = False,
  446. ) -> tuple[int, str]: ...
  447. @overload
  448. async def mkstemp(
  449. suffix: bytes | None = None,
  450. prefix: bytes | None = None,
  451. dir: bytes | None = None,
  452. text: bool = False,
  453. ) -> tuple[int, bytes]: ...
  454. async def mkstemp(
  455. suffix: AnyStr | None = None,
  456. prefix: AnyStr | None = None,
  457. dir: AnyStr | None = None,
  458. text: bool = False,
  459. ) -> tuple[int, str | bytes]:
  460. """
  461. Asynchronously create a temporary file and return an OS-level handle and the file
  462. name.
  463. This function wraps `tempfile.mkstemp` and executes it in a background thread.
  464. :param suffix: Suffix to be added to the file name.
  465. :param prefix: Prefix to be added to the file name.
  466. :param dir: Directory in which the temporary file is created.
  467. :param text: Whether the file is opened in text mode.
  468. :return: A tuple containing the file descriptor and the file name.
  469. """
  470. return await to_thread.run_sync(tempfile.mkstemp, suffix, prefix, dir, text)
  471. @overload
  472. async def mkdtemp(
  473. suffix: str | None = None,
  474. prefix: str | None = None,
  475. dir: str | None = None,
  476. ) -> str: ...
  477. @overload
  478. async def mkdtemp(
  479. suffix: bytes | None = None,
  480. prefix: bytes | None = None,
  481. dir: bytes | None = None,
  482. ) -> bytes: ...
  483. async def mkdtemp(
  484. suffix: AnyStr | None = None,
  485. prefix: AnyStr | None = None,
  486. dir: AnyStr | None = None,
  487. ) -> str | bytes:
  488. """
  489. Asynchronously create a temporary directory and return its path.
  490. This function wraps `tempfile.mkdtemp` and executes it in a background thread.
  491. :param suffix: Suffix to be added to the directory name.
  492. :param prefix: Prefix to be added to the directory name.
  493. :param dir: Parent directory where the temporary directory is created.
  494. :return: The path of the created temporary directory.
  495. """
  496. return await to_thread.run_sync(tempfile.mkdtemp, suffix, prefix, dir)
  497. async def gettempdir() -> str:
  498. """
  499. Asynchronously return the name of the directory used for temporary files.
  500. This function wraps `tempfile.gettempdir` and executes it in a background thread.
  501. :return: The path of the temporary directory as a string.
  502. """
  503. return await to_thread.run_sync(tempfile.gettempdir)
  504. async def gettempdirb() -> bytes:
  505. """
  506. Asynchronously return the name of the directory used for temporary files in bytes.
  507. This function wraps `tempfile.gettempdirb` and executes it in a background thread.
  508. :return: The path of the temporary directory as bytes.
  509. """
  510. return await to_thread.run_sync(tempfile.gettempdirb)