_wkt_registry.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599
  1. # Copyright (c) 2025-2026 Buf Technologies, Inc.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. """Central registry for matching well-known type descriptors.
  15. Consolidates WKT identification and field extraction into a single source
  16. of truth, replacing scattered string comparisons and cast chains.
  17. """
  18. from __future__ import annotations
  19. import re
  20. from copy import copy
  21. from dataclasses import dataclass
  22. from datetime import datetime, timezone
  23. from typing import TYPE_CHECKING, TypeAlias, cast
  24. from ._descriptors import (
  25. DescEnum,
  26. DescField,
  27. DescFieldValueEnum,
  28. DescFieldValueList,
  29. DescFieldValueMap,
  30. DescFieldValueMessage,
  31. DescFieldValueScalar,
  32. DescMessage,
  33. ScalarType,
  34. )
  35. if TYPE_CHECKING:
  36. from types import NotImplementedType
  37. from ._from_json import FromJsonOptions
  38. from ._message import Message
  39. from ._to_json import ToJsonOptions
  40. from ._typing import JsonValue
  41. from .wkt import Any, Duration, FieldMask, ListValue, Struct, Timestamp, Value
  42. from .wkt._gen.any_pb import Any as GenAny
  43. _TS_MIN = datetime(1, 1, 1, tzinfo=timezone.utc)
  44. _TS_MAX = datetime(9999, 12, 31, 23, 59, 59, tzinfo=timezone.utc)
  45. # Strict RFC 3339 validation — datetime.fromisoformat is too permissive
  46. # (accepts lowercase 't', space separator, offsets without colons).
  47. _TIMESTAMP_RE = re.compile(
  48. r"^([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2})"
  49. r"(?:\.([0-9]{1,9}))?"
  50. r"(Z|(?:[+-][0-9]{2}:[0-9]{2}))$"
  51. )
  52. _DURATION_RE = re.compile(r"^(-?[0-9]+)(?:\.([0-9]{1,9}))?s$")
  53. @dataclass(frozen=True, slots=True)
  54. class WktTimestamp:
  55. def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue:
  56. from .wkt import Timestamp # noqa: PLC0415
  57. from .wkt._mixin._const import ( # noqa: PLC0415
  58. DURATION_SECONDS_MAX,
  59. NANOS_PER_SECOND_MAX,
  60. )
  61. value = cast("Timestamp", msg)
  62. if not (-DURATION_SECONDS_MAX <= value.seconds <= DURATION_SECONDS_MAX):
  63. err = "timestamp seconds out of range"
  64. raise ValueError(err)
  65. if not (0 <= value.nanos <= NANOS_PER_SECOND_MAX):
  66. err = "timestamp nanos out of range"
  67. raise ValueError(err)
  68. # Use nanos=0 for datetime conversion — we append nanos separately
  69. # to preserve nanosecond precision.
  70. iso_secs = (
  71. Timestamp(seconds=value.seconds)
  72. .to_datetime()
  73. .isoformat(timespec="seconds")
  74. .removesuffix("+00:00")
  75. )
  76. if value.nanos == 0:
  77. return iso_secs + "Z"
  78. nanos_str = f"{value.nanos:09d}"
  79. if nanos_str[3:] == "000000":
  80. nanos_str = nanos_str[:3]
  81. elif nanos_str[6:] == "000":
  82. nanos_str = nanos_str[:6]
  83. return f"{iso_secs}.{nanos_str}Z"
  84. def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool:
  85. from .wkt import Timestamp # noqa: PLC0415
  86. value = cast("Timestamp", msg)
  87. if not isinstance(json, str):
  88. err = f"cannot decode {value._desc.type_name} from JSON: {json}"
  89. raise TypeError(err)
  90. matches = _TIMESTAMP_RE.match(json)
  91. if not matches:
  92. err = f"cannot decode {value._desc.type_name} from JSON: invalid RFC 3339 string"
  93. raise ValueError(err)
  94. nanos = 0
  95. if matches[2]:
  96. frac = matches[2]
  97. nanos = int("1" + frac + "0" * (9 - len(frac))) - 1_000_000_000
  98. # Reconstruct without fractional seconds for datetime parsing.
  99. # TODO: Remove this normalization when the Python floor is 3.11+.
  100. # Python 3.10's datetime.fromisoformat does not accept a trailing "Z".
  101. offset = "+00:00" if matches[3] == "Z" else matches[3]
  102. try:
  103. dt = datetime.fromisoformat(matches[1] + offset)
  104. except ValueError:
  105. err = f"cannot decode {value._desc.type_name} from JSON: invalid RFC 3339 string"
  106. raise ValueError(err) from None
  107. if dt < _TS_MIN or dt > _TS_MAX:
  108. err = (
  109. f"cannot decode {value._desc.type_name} from JSON: must be from"
  110. " 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive"
  111. )
  112. raise ValueError(err)
  113. value.seconds = Timestamp.from_datetime(dt).seconds
  114. value.nanos = nanos
  115. return True
  116. def mixin(self) -> type | None:
  117. from .wkt._mixin import TimestampMixin # noqa: PLC0415
  118. return TimestampMixin
  119. @dataclass(frozen=True, slots=True)
  120. class WktDuration:
  121. def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue:
  122. from .wkt._mixin._const import ( # noqa: PLC0415
  123. DURATION_SECONDS_MAX,
  124. NANOS_PER_SECOND_MAX,
  125. )
  126. value = cast("Duration", msg)
  127. if not (-DURATION_SECONDS_MAX <= value.seconds <= DURATION_SECONDS_MAX):
  128. err = "duration seconds out of range"
  129. raise ValueError(err)
  130. if not (-NANOS_PER_SECOND_MAX <= value.nanos <= NANOS_PER_SECOND_MAX):
  131. err = "duration nanos out of range"
  132. raise ValueError(err)
  133. if (value.seconds > 0 and value.nanos < 0) or (
  134. value.seconds < 0 and value.nanos > 0
  135. ):
  136. err = "duration seconds and nanos have different signs"
  137. raise ValueError(err)
  138. if value.nanos == 0:
  139. return f"{value.seconds}s"
  140. nanos_str = f"{abs(value.nanos):09d}"
  141. if nanos_str[3:] == "000000":
  142. nanos_str = nanos_str[:3]
  143. elif nanos_str[6:] == "000":
  144. nanos_str = nanos_str[:6]
  145. text = f"{value.seconds}.{nanos_str}"
  146. if value.nanos < 0 and value.seconds == 0:
  147. text = "-" + text
  148. return text + "s"
  149. def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool:
  150. from .wkt._mixin._const import DURATION_SECONDS_MAX # noqa: PLC0415
  151. value = cast("Duration", msg)
  152. if not isinstance(json, str):
  153. err = f"cannot decode {value._desc.type_name} from JSON: {json}"
  154. raise TypeError(err)
  155. duration_match = _DURATION_RE.match(json)
  156. if not duration_match:
  157. err = f"cannot decode {value._desc.type_name} from JSON: {json}"
  158. raise ValueError(err)
  159. seconds = int(duration_match[1])
  160. if seconds > DURATION_SECONDS_MAX or seconds < -DURATION_SECONDS_MAX:
  161. err = f"cannot decode {value._desc.type_name} from JSON: {json}"
  162. raise ValueError(err)
  163. nanos = 0
  164. if duration_match[2]:
  165. nanos = int(duration_match[2] + "0" * (9 - len(duration_match[2])))
  166. if seconds < 0 or duration_match[1] == "-0":
  167. nanos = -nanos
  168. value.seconds = seconds
  169. value.nanos = nanos
  170. return True
  171. def mixin(self) -> type | None:
  172. from .wkt._mixin import DurationMixin # noqa: PLC0415
  173. return DurationMixin
  174. @dataclass(frozen=True, slots=True)
  175. class WktAny:
  176. def to_json_value(self, msg: Message, opts: ToJsonOptions) -> JsonValue:
  177. from ._to_json import _message_to_json_value # noqa: PLC0415
  178. from .wkt._mixin._any import type_url_to_name # noqa: PLC0415
  179. message = cast("GenAny", msg)
  180. if message.type_url == "":
  181. return {}
  182. if not opts.registry:
  183. err = f'any "{message.type_url}" is not in the type registry'
  184. raise ValueError(err)
  185. type_name = type_url_to_name(message.type_url)
  186. desc = opts.registry.message(type_name)
  187. if not desc:
  188. err = f'any: "{message.type_url}" is not in the type registry'
  189. raise ValueError(err)
  190. unpacked = message.unpack(desc)
  191. assert unpacked is not None # noqa: S101
  192. json_value = _message_to_json_value(unpacked, opts)
  193. # WKTs have custom JSON representations (strings, arrays, null, etc.)
  194. # that must be wrapped in a "value" field. Regular messages produce
  195. # dicts whose fields are merged alongside "@type".
  196. if not isinstance(json_value, dict) or match_wkt(desc) is not None:
  197. return {"@type": message.type_url, "value": json_value}
  198. json_value["@type"] = message.type_url
  199. return json_value
  200. def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool:
  201. from ._from_json import _read_message # noqa: PLC0415
  202. from .wkt import Any # noqa: PLC0415
  203. value = cast("Any", msg)
  204. if not isinstance(json, dict):
  205. err = f"cannot decode {Any._desc.type_name} from JSON: {json}"
  206. raise TypeError(err)
  207. if not json:
  208. return True
  209. type_url = json.get("@type")
  210. if not isinstance(type_url, str) or not type_url:
  211. err = f"cannot decode {Any._desc.type_name} from JSON: {json}, @type is invalid: {type_url}"
  212. raise ValueError(err)
  213. type_name = (
  214. type_url[type_url.rindex("/") + 1 :] if "/" in type_url else type_url
  215. )
  216. desc = opts.registry.message(type_name) if opts.registry else None
  217. if not desc:
  218. err = f"cannot decode {Any._desc.type_name} from JSON: {type_url} is not in the type registry"
  219. raise ValueError(err)
  220. message = desc.type()
  221. if match_wkt(desc) is not None and "value" in json:
  222. _read_message(message, json["value"], opts)
  223. else:
  224. json = copy(json)
  225. del json["@type"]
  226. _read_message(message, json, opts)
  227. any_ = Any.pack(message)
  228. value.type_url = any_.type_url
  229. value.value = any_.value
  230. return True
  231. def mixin(self) -> type | None:
  232. from .wkt._mixin import AnyMixin # noqa: PLC0415
  233. return AnyMixin
  234. @dataclass(frozen=True, slots=True)
  235. class WktFieldMask:
  236. def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue:
  237. from ._names import proto_camel_case, proto_snake_case # noqa: PLC0415
  238. value = cast("FieldMask", msg)
  239. parts: list[str] = []
  240. for path in value.paths:
  241. proto_camel_path = proto_camel_case(path)
  242. if proto_snake_case(proto_camel_path) != path:
  243. err = (
  244. f"invalid FieldMask path: lowerCamelCase of {path} is irreversible"
  245. )
  246. raise ValueError(err)
  247. parts.append(proto_camel_path)
  248. return ",".join(parts)
  249. def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool:
  250. from ._names import proto_snake_case # noqa: PLC0415
  251. value = cast("FieldMask", msg)
  252. if not isinstance(json, str):
  253. err = f"cannot decode {value._desc.type_name} from JSON: {json}"
  254. raise TypeError(err)
  255. if not json:
  256. return True
  257. for path in json.split(","):
  258. if "_" in path:
  259. err = f"cannot decode {value._desc.type_name} from JSON: path names must be lowerCamelCase"
  260. raise ValueError(err)
  261. value.paths.append(proto_snake_case(path))
  262. return True
  263. def mixin(self) -> type | None:
  264. return None
  265. @dataclass(frozen=True, slots=True)
  266. class WktStruct:
  267. fields: DescFieldValueMap
  268. def to_json_value(self, msg: Message, opts: ToJsonOptions) -> JsonValue:
  269. from ._to_json import _struct_to_json_value # noqa: PLC0415
  270. return _struct_to_json_value(cast("Struct", msg), opts)
  271. def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool:
  272. from ._from_json import _struct_from_json # noqa: PLC0415
  273. _struct_from_json(cast("Struct", msg), json, opts, self.fields)
  274. return True
  275. def mixin(self) -> type | None:
  276. return None
  277. @dataclass(frozen=True, slots=True)
  278. class WktListValue:
  279. values: DescFieldValueList
  280. def to_json_value(self, msg: Message, opts: ToJsonOptions) -> JsonValue:
  281. from ._to_json import _value_to_json_value # noqa: PLC0415
  282. return [
  283. _value_to_json_value(element, opts)
  284. for element in cast("ListValue", msg).values
  285. ]
  286. def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool:
  287. from ._from_json import _list_value_from_json # noqa: PLC0415
  288. _list_value_from_json(cast("ListValue", msg), json, opts, self.values)
  289. return True
  290. def mixin(self) -> type | None:
  291. return None
  292. @dataclass(frozen=True, slots=True)
  293. class WktValue:
  294. null_value: DescFieldValueEnum
  295. number_value: DescFieldValueScalar
  296. string_value: DescFieldValueScalar
  297. bool_value: DescFieldValueScalar
  298. struct_value: DescFieldValueMessage
  299. list_value: DescFieldValueMessage
  300. def to_json_value(self, msg: Message, opts: ToJsonOptions) -> JsonValue:
  301. from ._to_json import _value_to_json_value # noqa: PLC0415
  302. return _value_to_json_value(cast("Value", msg), opts)
  303. def from_json(self, msg: Message, json: JsonValue, opts: FromJsonOptions) -> bool:
  304. from ._from_json import _value_from_json # noqa: PLC0415
  305. _value_from_json(cast("Value", msg), json, opts, self)
  306. return True
  307. def mixin(self) -> type | None:
  308. return None
  309. @dataclass(frozen=True, slots=True)
  310. class WktWrapper:
  311. field: DescField
  312. value: DescFieldValueScalar
  313. def to_json_value(self, msg: Message, _opts: ToJsonOptions) -> JsonValue:
  314. from ._to_json import _scalar_to_json_value # noqa: PLC0415
  315. return _scalar_to_json_value(self.value.scalar, msg[self.field])
  316. def from_json(self, msg: Message, json: JsonValue, _opts: FromJsonOptions) -> bool:
  317. from ._from_json import _read_scalar # noqa: PLC0415
  318. if json is None:
  319. del msg[self.field]
  320. else:
  321. msg[self.field] = _read_scalar(self.field, self.value.scalar, json)
  322. return True
  323. def mixin(self) -> type | None:
  324. return None
  325. @dataclass(frozen=True, slots=True)
  326. class WktFileDescriptorSet:
  327. def to_json_value(self, _msg: Message, _opts: ToJsonOptions) -> NotImplementedType:
  328. return NotImplemented
  329. def from_json(
  330. self, _msg: Message, _json: JsonValue, _opts: FromJsonOptions
  331. ) -> bool:
  332. return False
  333. def mixin(self) -> type | None:
  334. from .wkt._mixin import FileDescriptorSetMixin # noqa: PLC0415
  335. return FileDescriptorSetMixin
  336. WktMatch: TypeAlias = (
  337. WktTimestamp
  338. | WktDuration
  339. | WktAny
  340. | WktFieldMask
  341. | WktStruct
  342. | WktListValue
  343. | WktValue
  344. | WktWrapper
  345. | WktFileDescriptorSet
  346. )
  347. def match_wkt(desc: DescMessage) -> WktMatch | None:
  348. """Match a message descriptor against known well-known types.
  349. Returns a typed match object with pre-extracted field descriptors if the
  350. descriptor's file name, type name, and fields match a known WKT. Returns
  351. None otherwise.
  352. """
  353. if not desc.type_name.startswith("google.protobuf."):
  354. return None
  355. if not desc.file.name.startswith("google/protobuf/"):
  356. return None
  357. match desc.type_name:
  358. case "google.protobuf.Timestamp":
  359. return _match_timestamp(desc)
  360. case "google.protobuf.Duration":
  361. return _match_duration(desc)
  362. case "google.protobuf.Any":
  363. return _match_any(desc)
  364. case "google.protobuf.FieldMask":
  365. return _match_field_mask(desc)
  366. case "google.protobuf.Struct":
  367. return _match_struct(desc)
  368. case "google.protobuf.ListValue":
  369. return _match_list_value(desc)
  370. case "google.protobuf.Value":
  371. return _match_value(desc)
  372. case "google.protobuf.FileDescriptorSet":
  373. return _match_file_descriptor_set(desc)
  374. case _:
  375. return _match_wrapper(desc)
  376. def is_wkt_value(desc: DescMessage) -> bool:
  377. """Check if a message descriptor is google.protobuf.Value."""
  378. return desc.type_name == "google.protobuf.Value" and desc.file.name.startswith(
  379. "google/protobuf/"
  380. )
  381. def is_null_value_enum(desc: DescEnum) -> bool:
  382. """Check if an enum descriptor is google.protobuf.NullValue."""
  383. return desc.type_name == "google.protobuf.NullValue" and desc.file.name.startswith(
  384. "google/protobuf/"
  385. )
  386. # The type name in the outer match of match_wkt already discriminates
  387. # Timestamp from Duration. The field checks below are a defensive guard
  388. # to reject descriptors whose fields don't match the expected schema.
  389. def _match_timestamp(desc: DescMessage) -> WktTimestamp | None:
  390. fields = desc._fields_by_name
  391. if (
  392. (seconds := fields.get("seconds"))
  393. and isinstance(seconds.value, DescFieldValueScalar)
  394. and seconds.value.scalar == ScalarType.INT64
  395. and (nanos := fields.get("nanos"))
  396. and isinstance(nanos.value, DescFieldValueScalar)
  397. and nanos.value.scalar == ScalarType.INT32
  398. ):
  399. return WktTimestamp()
  400. return None
  401. def _match_duration(desc: DescMessage) -> WktDuration | None:
  402. fields = desc._fields_by_name
  403. if (
  404. (seconds := fields.get("seconds"))
  405. and isinstance(seconds.value, DescFieldValueScalar)
  406. and seconds.value.scalar == ScalarType.INT64
  407. and (nanos := fields.get("nanos"))
  408. and isinstance(nanos.value, DescFieldValueScalar)
  409. and nanos.value.scalar == ScalarType.INT32
  410. ):
  411. return WktDuration()
  412. return None
  413. def _match_any(desc: DescMessage) -> WktAny | None:
  414. fields = desc._fields_by_name
  415. if (
  416. (type_url := fields.get("type_url"))
  417. and isinstance(type_url.value, DescFieldValueScalar)
  418. and type_url.value.scalar == ScalarType.STRING
  419. and (value := fields.get("value"))
  420. and isinstance(value.value, DescFieldValueScalar)
  421. and value.value.scalar == ScalarType.BYTES
  422. ):
  423. return WktAny()
  424. return None
  425. def _match_field_mask(desc: DescMessage) -> WktFieldMask | None:
  426. fields = desc._fields_by_name
  427. if (
  428. (paths := fields.get("paths"))
  429. and isinstance(paths.value, DescFieldValueList)
  430. and paths.value.element == ScalarType.STRING
  431. ):
  432. return WktFieldMask()
  433. return None
  434. def _match_struct(desc: DescMessage) -> WktStruct | None:
  435. fields = desc._fields_by_name
  436. if (
  437. (f := fields.get("fields"))
  438. and isinstance(f.value, DescFieldValueMap)
  439. and f.value.key == ScalarType.STRING
  440. ):
  441. return WktStruct(fields=f.value)
  442. return None
  443. def _match_list_value(desc: DescMessage) -> WktListValue | None:
  444. fields = desc._fields_by_name
  445. if (
  446. (values := fields.get("values"))
  447. and isinstance(values.value, DescFieldValueList)
  448. and isinstance(values.value.element, DescMessage)
  449. ):
  450. return WktListValue(values=values.value)
  451. return None
  452. def _match_value(desc: DescMessage) -> WktValue | None:
  453. fields = desc._fields_by_name
  454. if (
  455. (null_value := fields.get("null_value"))
  456. and isinstance(null_value.value, DescFieldValueEnum)
  457. and (number_value := fields.get("number_value"))
  458. and isinstance(number_value.value, DescFieldValueScalar)
  459. and number_value.value.scalar == ScalarType.DOUBLE
  460. and (string_value := fields.get("string_value"))
  461. and isinstance(string_value.value, DescFieldValueScalar)
  462. and string_value.value.scalar == ScalarType.STRING
  463. and (bool_value := fields.get("bool_value"))
  464. and isinstance(bool_value.value, DescFieldValueScalar)
  465. and bool_value.value.scalar == ScalarType.BOOL
  466. and (struct_value := fields.get("struct_value"))
  467. and isinstance(struct_value.value, DescFieldValueMessage)
  468. and (list_value := fields.get("list_value"))
  469. and isinstance(list_value.value, DescFieldValueMessage)
  470. ):
  471. return WktValue(
  472. null_value=null_value.value,
  473. number_value=number_value.value,
  474. string_value=string_value.value,
  475. bool_value=bool_value.value,
  476. struct_value=struct_value.value,
  477. list_value=list_value.value,
  478. )
  479. return None
  480. def _match_file_descriptor_set(desc: DescMessage) -> WktFileDescriptorSet | None:
  481. fields = desc._fields_by_name
  482. if (
  483. (file := fields.get("file"))
  484. and isinstance(file.value, DescFieldValueList)
  485. and isinstance(file.value.element, DescMessage)
  486. ):
  487. return WktFileDescriptorSet()
  488. return None
  489. def _match_wrapper(desc: DescMessage) -> WktWrapper | None:
  490. # Uses structural matching rather than an explicit allowlist of wrapper
  491. # type names. Any google.protobuf.* message with exactly one scalar field
  492. # named "value" is treated as a wrapper type. This is safe because all
  493. # other known WKT types are matched explicitly above in match_wkt.
  494. if len(desc.fields) != 1:
  495. return None
  496. field = desc.fields[0]
  497. if not isinstance(field.value, DescFieldValueScalar) or field.name != "value":
  498. return None
  499. return WktWrapper(field=field, value=field.value)