_binary_writer.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. # Copyright (c) 2025-2026 Buf Technologies, Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import annotations
  15. import struct
  16. from typing import TYPE_CHECKING, final
  17. if TYPE_CHECKING:
  18. from ._wire_type import WireType
  19. def _append_varint(buf: bytearray, value: int) -> None:
  20. """Append varint-encoded bytes to buf."""
  21. while value > 0x7F:
  22. buf.append((value & 0x7F) | 0x80)
  23. value >>= 7
  24. buf.append(value & 0x7F)
  25. @final
  26. class BinaryWriter:
  27. """A writer for serializing Protocol Buffer wire format.
  28. Length-delimited message fields are handled with `fork()`/`join()` pairs.
  29. Each `fork()` opens a new scope — the writing context for one sub-message —
  30. and reserves a placeholder for the length varint whose value is not yet
  31. known. Each `join()` closes the scope, fills in the placeholder with the
  32. actual byte count, and returns to the enclosing scope.
  33. All scopes share a single flat chunk list. Each chunk is appended exactly
  34. once and never moved or copied, giving O(n) total work where n is the
  35. number of bytes serialized.
  36. """
  37. def __init__(self) -> None:
  38. # The output of the writer will be the concatenation of all chunks in this list.
  39. self._chunks: list[bytes | bytearray] = []
  40. # Accumulation buffer for small writes; flushed to _chunks at structural boundaries.
  41. self._buf: bytearray = bytearray()
  42. # Number of bytes written in the current scope (resets to 0 on fork()).
  43. self._size: int = 0
  44. # Stack of (placeholder_idx, parent_size): pushed by fork(), popped by join().
  45. self._stack: list[tuple[int, int]] = []
  46. def _flush(self) -> None:
  47. """Move the accumulation buffer into _chunks if non-empty, updating size."""
  48. if self._buf:
  49. self._size += len(self._buf)
  50. self._chunks.append(self._buf)
  51. self._buf = bytearray()
  52. def fork(self) -> None:
  53. """Open a new scope for a length-delimited sub-message field.
  54. Reserves a placeholder for the length varint and saves the current
  55. scope on the stack. Subsequent writes accumulate into the new scope.
  56. Must be paired with a call to join().
  57. """
  58. self._flush()
  59. placeholder_idx = len(self._chunks)
  60. self._chunks.append(b"") # placeholder for the length varint
  61. self._stack.append((placeholder_idx, self._size))
  62. self._size = 0
  63. def join(self) -> None:
  64. """Close the current scope and finalize the sub-message length.
  65. Fills the placeholder reserved by `fork()` with the actual byte count of
  66. the scope, then restores the enclosing scope.
  67. """
  68. if not self._stack:
  69. msg = "join() called without a matching fork()"
  70. raise RuntimeError(msg)
  71. self._flush()
  72. placeholder_idx, parent_size = self._stack.pop()
  73. sub_size = self._size
  74. length_varint = bytearray()
  75. _append_varint(length_varint, sub_size)
  76. self._chunks[placeholder_idx] = length_varint
  77. self._size = parent_size + len(length_varint) + sub_size
  78. def finish(self) -> bytes:
  79. """Return the serialized bytes.
  80. Returns:
  81. The complete serialized message as bytes.
  82. """
  83. if self._stack:
  84. msg = f"finish() called with {len(self._stack)} unclosed fork()"
  85. raise RuntimeError(msg)
  86. self._flush()
  87. return b"".join(self._chunks)
  88. def _varint(self, value: int) -> None:
  89. _append_varint(self._buf, value)
  90. def tag(self, number: int, wire_type: WireType) -> None:
  91. """Write a field tag (field number + wire type).
  92. Args:
  93. number: The field number.
  94. wire_type: The wire type for the field.
  95. """
  96. self.uint32((number << 3) | wire_type)
  97. def bool_(self, value: bool) -> None: # noqa: FBT001
  98. """Write a boolean as a varint.
  99. Args:
  100. value: The boolean to encode.
  101. """
  102. self._varint(int(value))
  103. def int32(self, value: int) -> None:
  104. """Write a signed 32-bit integer as a varint.
  105. Negative values are encoded using two's complement.
  106. Args:
  107. value: The signed 32-bit integer to encode.
  108. """
  109. if value < -(1 << 31) or value >= (1 << 31):
  110. msg = f"value out of range for int32: {value}"
  111. raise ValueError(msg)
  112. self._varint(value % (1 << 64))
  113. def int64(self, value: int) -> None:
  114. """Write a signed 64-bit integer as a varint.
  115. Negative values are encoded using two's complement.
  116. Args:
  117. value: The signed 64-bit integer to encode.
  118. """
  119. if value < -(1 << 63) or value >= (1 << 63):
  120. msg = f"value out of range for int64: {value}"
  121. raise ValueError(msg)
  122. self._varint(value % (1 << 64))
  123. def uint32(self, value: int) -> None:
  124. """Write an unsigned 32-bit integer as a varint.
  125. Args:
  126. value: The unsigned 32-bit integer to encode.
  127. """
  128. if value < 0 or value >= (1 << 32):
  129. msg = f"value out of range for uint32: {value}"
  130. raise ValueError(msg)
  131. self._varint(value)
  132. def uint64(self, value: int) -> None:
  133. """Write an unsigned 64-bit integer as a varint.
  134. Args:
  135. value: The unsigned 64-bit integer to encode.
  136. """
  137. if value < 0 or value >= (1 << 64):
  138. msg = f"value out of range for uint64: {value}"
  139. raise ValueError(msg)
  140. self._varint(value)
  141. def sint32(self, value: int) -> None:
  142. """Write a signed 32-bit integer as a zigzag-encoded varint.
  143. Args:
  144. value: The signed 32-bit integer to encode.
  145. """
  146. if value < -(1 << 31) or value >= (1 << 31):
  147. msg = f"value out of range for sint32: {value}"
  148. raise ValueError(msg)
  149. self._varint((value << 1) ^ (value >> 31))
  150. def sint64(self, value: int) -> None:
  151. """Write a signed 64-bit integer as a zigzag-encoded varint.
  152. Args:
  153. value: The signed 64-bit integer to encode.
  154. """
  155. if value < -(1 << 63) or value >= (1 << 63):
  156. msg = f"value out of range for sint64: {value}"
  157. raise ValueError(msg)
  158. self._varint((value << 1) ^ (value >> 63))
  159. def fixed32(self, value: int) -> None:
  160. """Write an unsigned 32-bit integer in fixed-width format.
  161. Args:
  162. value: The unsigned 32-bit integer to encode.
  163. """
  164. if value < 0 or value >= (1 << 32):
  165. msg = f"value out of range for fixed32: {value}"
  166. raise ValueError(msg)
  167. self._buf += struct.pack("<I", value)
  168. def sfixed32(self, value: int) -> None:
  169. """Write a signed 32-bit integer in fixed-width format.
  170. Args:
  171. value: The signed 32-bit integer to encode.
  172. """
  173. if value < -(1 << 31) or value >= (1 << 31):
  174. msg = f"value out of range for sfixed32: {value}"
  175. raise ValueError(msg)
  176. self._buf += struct.pack("<i", value)
  177. def fixed64(self, value: int) -> None:
  178. """Write an unsigned 64-bit integer in fixed-width format.
  179. Args:
  180. value: The unsigned 64-bit integer to encode.
  181. """
  182. if value < 0 or value >= (1 << 64):
  183. msg = f"value out of range for fixed64: {value}"
  184. raise ValueError(msg)
  185. self._buf += struct.pack("<Q", value)
  186. def sfixed64(self, value: int) -> None:
  187. """Write a signed 64-bit integer in fixed-width format.
  188. Args:
  189. value: The signed 64-bit integer to encode.
  190. """
  191. if value < -(1 << 63) or value >= (1 << 63):
  192. msg = f"value out of range for sfixed64: {value}"
  193. raise ValueError(msg)
  194. self._buf += struct.pack("<q", value)
  195. def float_(self, value: float) -> None:
  196. """Write a 32-bit floating point number.
  197. Args:
  198. value: The float to encode.
  199. """
  200. self._buf += struct.pack("<f", value)
  201. def double(self, value: float) -> None:
  202. """Write a 64-bit floating point number.
  203. Args:
  204. value: The float to encode.
  205. """
  206. self._buf += struct.pack("<d", value)
  207. def bytes_(self, value: bytes) -> None:
  208. """Write a length-delimited byte sequence.
  209. Args:
  210. value: The bytes to encode.
  211. """
  212. self._varint(len(value))
  213. self._flush()
  214. self._size += len(value)
  215. self._chunks.append(value)
  216. def raw(self, value: bytes) -> None:
  217. """Write raw bytes directly to the output without any encoding.
  218. Args:
  219. value: The raw bytes to write.
  220. """
  221. self._flush()
  222. self._size += len(value)
  223. self._chunks.append(value)
  224. def string(self, value: str) -> None:
  225. """Write a length-delimited UTF-8 string.
  226. Args:
  227. value: The string to encode.
  228. """
  229. self.bytes_(value.encode("utf-8"))