json_format.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137
  1. # Protocol Buffers - Google's data interchange format
  2. # Copyright 2008 Google Inc. All rights reserved.
  3. #
  4. # Use of this source code is governed by a BSD-style
  5. # license that can be found in the LICENSE file or at
  6. # https://developers.google.com/open-source/licenses/bsd
  7. """Contains routines for printing protocol messages in JSON format.
  8. Simple usage example:
  9. # Create a proto object and serialize it to a json format string.
  10. message = my_proto_pb2.MyMessage(foo='bar')
  11. json_string = json_format.MessageToJson(message)
  12. # Parse a json format string to proto object.
  13. message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
  14. """
  15. __author__ = 'jieluo@google.com (Jie Luo)'
  16. import base64
  17. from collections import OrderedDict
  18. import json
  19. import math
  20. from operator import methodcaller
  21. import re
  22. from google.protobuf import descriptor
  23. from google.protobuf import descriptor_pool
  24. from google.protobuf import message_factory
  25. from google.protobuf import symbol_database
  26. from google.protobuf.internal import type_checkers
  27. _INT_TYPES = frozenset([
  28. descriptor.FieldDescriptor.CPPTYPE_INT32,
  29. descriptor.FieldDescriptor.CPPTYPE_UINT32,
  30. descriptor.FieldDescriptor.CPPTYPE_INT64,
  31. descriptor.FieldDescriptor.CPPTYPE_UINT64,
  32. ])
  33. _INT64_TYPES = frozenset([
  34. descriptor.FieldDescriptor.CPPTYPE_INT64,
  35. descriptor.FieldDescriptor.CPPTYPE_UINT64,
  36. ])
  37. _FLOAT_TYPES = frozenset([
  38. descriptor.FieldDescriptor.CPPTYPE_FLOAT,
  39. descriptor.FieldDescriptor.CPPTYPE_DOUBLE,
  40. ])
  41. _INFINITY = 'Infinity'
  42. _NEG_INFINITY = '-Infinity'
  43. _NAN = 'NaN'
  44. _UNPAIRED_SURROGATE_PATTERN = re.compile(
  45. '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
  46. )
  47. _VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
  48. class Error(Exception):
  49. """Top-level module error for json_format."""
  50. class SerializeToJsonError(Error):
  51. """Thrown if serialization to JSON fails."""
  52. class ParseError(Error):
  53. """Thrown in case of parsing error."""
  54. class EnumStringValueParseError(ParseError):
  55. """Thrown if unknown string enum value is encountered.
  56. This exception is suppressed if ignore_unknown_fields is set.
  57. """
  58. def MessageToJson(
  59. message,
  60. preserving_proto_field_name=False,
  61. indent=2,
  62. sort_keys=False,
  63. use_integers_for_enums=False,
  64. descriptor_pool=None,
  65. ensure_ascii=True,
  66. always_print_fields_with_no_presence=False,
  67. *,
  68. unquote_int64_if_possible=False,
  69. ):
  70. """Converts protobuf message to JSON format.
  71. Args:
  72. message: The protocol buffers message instance to serialize.
  73. always_print_fields_with_no_presence: If True, fields without presence
  74. (implicit presence scalars, repeated fields, and map fields) will always
  75. be serialized. Any field that supports presence is not affected by this
  76. option (including singular message fields and oneof fields).
  77. preserving_proto_field_name: If True, use the original proto field names as
  78. defined in the .proto file. If False, convert the field names to
  79. lowerCamelCase.
  80. indent: The JSON object will be pretty-printed with this indent level. An
  81. indent level of 0 or negative will only insert newlines. If the indent
  82. level is None, no newlines will be inserted.
  83. sort_keys: If True, then the output will be sorted by field names.
  84. use_integers_for_enums: If true, print integers instead of enum names.
  85. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  86. default.
  87. ensure_ascii: If True, strings with non-ASCII characters are escaped. If
  88. False, Unicode strings are returned unchanged.
  89. unquote_int64_if_possible: If True, unquote int64 fields for values that are
  90. safe to emit as numbers (all values smaller than 2^53 and a sparse set of
  91. values that are larger).
  92. Returns:
  93. A string containing the JSON formatted protocol buffer message.
  94. """
  95. printer = _Printer(
  96. preserving_proto_field_name,
  97. use_integers_for_enums,
  98. descriptor_pool,
  99. always_print_fields_with_no_presence,
  100. unquote_int64_if_possible=unquote_int64_if_possible,
  101. )
  102. return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
  103. def MessageToDict(
  104. message,
  105. always_print_fields_with_no_presence=False,
  106. preserving_proto_field_name=False,
  107. use_integers_for_enums=False,
  108. descriptor_pool=None,
  109. *,
  110. unquote_int64_if_possible=False,
  111. ):
  112. """Converts protobuf message to a dictionary.
  113. When the dictionary is encoded to JSON, it conforms to ProtoJSON spec.
  114. Args:
  115. message: The protocol buffers message instance to serialize.
  116. always_print_fields_with_no_presence: If True, fields without presence
  117. (implicit presence scalars, repeated fields, and map fields) will always
  118. be serialized. Any field that supports presence is not affected by this
  119. option (including singular message fields and oneof fields).
  120. preserving_proto_field_name: If True, use the original proto field names as
  121. defined in the .proto file. If False, convert the field names to
  122. lowerCamelCase.
  123. use_integers_for_enums: If true, print integers instead of enum names.
  124. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  125. default.
  126. unquote_int64_if_possible: If True, unquote int64 fields for values that are
  127. safe to emit as numbers (all values smaller than 2^53 and a sparse set of
  128. values that are larger).
  129. Returns:
  130. A dict representation of the protocol buffer message.
  131. """
  132. printer = _Printer(
  133. preserving_proto_field_name,
  134. use_integers_for_enums,
  135. descriptor_pool,
  136. always_print_fields_with_no_presence,
  137. unquote_int64_if_possible=unquote_int64_if_possible,
  138. )
  139. # pylint: disable=protected-access
  140. return printer._MessageToJsonObject(message)
  141. def _IsMapEntry(field):
  142. return (
  143. field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
  144. and field.message_type.has_options
  145. and field.message_type.GetOptions().map_entry
  146. )
  147. class _Printer(object):
  148. """JSON format printer for protocol message."""
  149. def __init__(
  150. self,
  151. preserving_proto_field_name=False,
  152. use_integers_for_enums=False,
  153. descriptor_pool=None,
  154. always_print_fields_with_no_presence=False,
  155. *,
  156. unquote_int64_if_possible=False,
  157. ):
  158. self.always_print_fields_with_no_presence = (
  159. always_print_fields_with_no_presence
  160. )
  161. self.preserving_proto_field_name = preserving_proto_field_name
  162. self.use_integers_for_enums = use_integers_for_enums
  163. self.descriptor_pool = descriptor_pool
  164. self.unquote_int64_if_possible = unquote_int64_if_possible
  165. def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
  166. js = self._MessageToJsonObject(message)
  167. return json.dumps(
  168. js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii
  169. )
  170. def _MessageToJsonObject(self, message):
  171. """Converts message to an object according to ProtoJSON Specification."""
  172. message_descriptor = message.DESCRIPTOR
  173. full_name = message_descriptor.full_name
  174. if _IsWrapperMessage(message_descriptor):
  175. return self._WrapperMessageToJsonObject(message)
  176. if full_name in _WKTJSONMETHODS:
  177. return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)
  178. js = {}
  179. return self._RegularMessageToJsonObject(message, js)
  180. def _RegularMessageToJsonObject(self, message, js):
  181. """Converts normal message according to ProtoJSON Specification."""
  182. fields = message.ListFields()
  183. try:
  184. for field, value in fields:
  185. if field.is_extension:
  186. name = '[%s]' % field.full_name
  187. elif self.preserving_proto_field_name:
  188. name = field.name
  189. else:
  190. name = field.json_name
  191. if _IsMapEntry(field):
  192. # Convert a map field.
  193. v_field = field.message_type.fields_by_name['value']
  194. js_map = {}
  195. for key in value:
  196. if isinstance(key, bool):
  197. if key:
  198. recorded_key = 'true'
  199. else:
  200. recorded_key = 'false'
  201. else:
  202. recorded_key = str(key)
  203. js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])
  204. js[name] = js_map
  205. elif field.is_repeated:
  206. # Convert a repeated field.
  207. js[name] = [self._FieldToJsonObject(field, k) for k in value]
  208. else:
  209. js[name] = self._FieldToJsonObject(field, value)
  210. # Serialize default value if including_default_value_fields is True.
  211. if (
  212. self.always_print_fields_with_no_presence
  213. ):
  214. message_descriptor = message.DESCRIPTOR
  215. for field in message_descriptor.fields:
  216. # always_print_fields_with_no_presence doesn't apply to
  217. # any field which supports presence.
  218. if self.always_print_fields_with_no_presence and field.has_presence:
  219. continue
  220. if self.preserving_proto_field_name:
  221. name = field.name
  222. else:
  223. name = field.json_name
  224. if name in js:
  225. # Skip the field which has been serialized already.
  226. continue
  227. if _IsMapEntry(field):
  228. js[name] = {}
  229. elif field.is_repeated:
  230. js[name] = []
  231. else:
  232. js[name] = self._FieldToJsonObject(field, field.default_value)
  233. except ValueError as e:
  234. raise SerializeToJsonError(
  235. 'Failed to serialize {0} field: {1}.'.format(field.name, e)
  236. ) from e
  237. return js
  238. def _FieldToJsonObject(self, field, value):
  239. """Converts field value according to ProtoJSON Specification."""
  240. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  241. return self._MessageToJsonObject(value)
  242. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  243. if self.use_integers_for_enums:
  244. return value
  245. if field.enum_type.full_name == 'google.protobuf.NullValue':
  246. return None
  247. enum_value = field.enum_type.values_by_number.get(value, None)
  248. if enum_value is not None:
  249. option = _GetJsonEnumValueOption(enum_value)
  250. if option is not None:
  251. return option.string
  252. return enum_value.name
  253. else:
  254. if field.enum_type.is_closed:
  255. raise SerializeToJsonError(
  256. 'Enum field contains an integer value '
  257. 'which can not mapped to an enum value.'
  258. )
  259. else:
  260. return value
  261. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  262. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  263. # Use base64 Data encoding for bytes
  264. return base64.b64encode(value).decode('utf-8')
  265. else:
  266. return str(value)
  267. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  268. return bool(value)
  269. elif field.cpp_type in _INT64_TYPES:
  270. if self.unquote_int64_if_possible and float(value) == value:
  271. return value
  272. else:
  273. return str(value)
  274. elif field.cpp_type in _FLOAT_TYPES:
  275. if math.isinf(value):
  276. if value < 0.0:
  277. return _NEG_INFINITY
  278. else:
  279. return _INFINITY
  280. if math.isnan(value):
  281. return _NAN
  282. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  283. return type_checkers.ToShortestFloat(value)
  284. return value
  285. def _AnyMessageToJsonObject(self, message):
  286. """Converts Any message according to ProtoJSON Specification."""
  287. if not message.ListFields():
  288. return {}
  289. # Must print @type first, use OrderedDict instead of {}
  290. js = OrderedDict()
  291. type_url = message.type_url
  292. js['@type'] = type_url
  293. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  294. sub_message.ParseFromString(message.value)
  295. message_descriptor = sub_message.DESCRIPTOR
  296. full_name = message_descriptor.full_name
  297. if _IsWrapperMessage(message_descriptor):
  298. js['value'] = self._WrapperMessageToJsonObject(sub_message)
  299. return js
  300. if full_name in _WKTJSONMETHODS:
  301. js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(
  302. self
  303. )
  304. return js
  305. return self._RegularMessageToJsonObject(sub_message, js)
  306. def _GenericMessageToJsonObject(self, message):
  307. """Converts message according to ProtoJSON Specification."""
  308. # Duration, Timestamp and FieldMask have ToJsonString method to do the
  309. # convert. Users can also call the method directly.
  310. return message.ToJsonString()
  311. def _ValueMessageToJsonObject(self, message):
  312. """Converts Value message according to ProtoJSON Specification."""
  313. which = message.WhichOneof('kind')
  314. # If the Value message is not set treat as null_value when serialize
  315. # to JSON. The parse back result will be different from original message.
  316. if which is None or which == 'null_value':
  317. return None
  318. if which == 'list_value':
  319. return self._ListValueMessageToJsonObject(message.list_value)
  320. if which == 'number_value':
  321. value = message.number_value
  322. if math.isinf(value):
  323. raise ValueError(
  324. 'Fail to serialize Infinity for Value.number_value, '
  325. 'which would parse as string_value'
  326. )
  327. if math.isnan(value):
  328. raise ValueError(
  329. 'Fail to serialize NaN for Value.number_value, '
  330. 'which would parse as string_value'
  331. )
  332. else:
  333. value = getattr(message, which)
  334. oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
  335. return self._FieldToJsonObject(oneof_descriptor, value)
  336. def _ListValueMessageToJsonObject(self, message):
  337. """Converts ListValue message according to ProtoJSON Specification."""
  338. return [self._ValueMessageToJsonObject(value) for value in message.values]
  339. def _StructMessageToJsonObject(self, message):
  340. """Converts Struct message according to ProtoJSON Specification."""
  341. fields = message.fields
  342. ret = {}
  343. for key in fields:
  344. ret[key] = self._ValueMessageToJsonObject(fields[key])
  345. return ret
  346. def _WrapperMessageToJsonObject(self, message):
  347. return self._FieldToJsonObject(
  348. message.DESCRIPTOR.fields_by_name['value'], message.value
  349. )
  350. def _IsWrapperMessage(message_descriptor):
  351. return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
  352. def _DuplicateChecker(js):
  353. result = {}
  354. for name, value in js:
  355. if name in result:
  356. raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
  357. result[name] = value
  358. return result
  359. def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
  360. """Creates a message from a type URL."""
  361. db = symbol_database.Default()
  362. pool = db.pool if descriptor_pool is None else descriptor_pool
  363. type_name = type_url.split('/')[-1]
  364. try:
  365. message_descriptor = pool.FindMessageTypeByName(type_name)
  366. except KeyError as e:
  367. raise TypeError(
  368. 'Can not find message descriptor by type_url: {0}'.format(type_url)
  369. ) from e
  370. message_class = message_factory.GetMessageClass(message_descriptor)
  371. return message_class()
  372. def Parse(
  373. text,
  374. message,
  375. ignore_unknown_fields=False,
  376. descriptor_pool=None,
  377. max_recursion_depth=100,
  378. ):
  379. """Parses a JSON representation of a protocol message into a message.
  380. Args:
  381. text: Message JSON representation.
  382. message: A protocol buffer message to merge into.
  383. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  384. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  385. default.
  386. max_recursion_depth: max recursion depth of JSON message to be deserialized.
  387. JSON messages over this depth will fail to be deserialized. Default value
  388. is 100.
  389. Returns:
  390. The same message passed as argument.
  391. Raises::
  392. ParseError: On JSON parsing problems.
  393. """
  394. if not isinstance(text, str):
  395. text = text.decode('utf-8')
  396. try:
  397. js = json.loads(text, object_pairs_hook=_DuplicateChecker)
  398. except Exception as e:
  399. raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
  400. try:
  401. return ParseDict(
  402. js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth
  403. )
  404. except ParseError as e:
  405. raise e
  406. except Exception as e:
  407. raise ParseError(
  408. 'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e))
  409. ) from e
  410. def ParseDict(
  411. js_dict,
  412. message,
  413. ignore_unknown_fields=False,
  414. descriptor_pool=None,
  415. max_recursion_depth=100,
  416. ):
  417. """Parses a JSON dictionary representation into a message.
  418. Args:
  419. js_dict: Dict representation of a JSON message.
  420. message: A protocol buffer message to merge into.
  421. ignore_unknown_fields: If True, do not raise errors for unknown fields.
  422. descriptor_pool: A Descriptor Pool for resolving types. If None use the
  423. default.
  424. max_recursion_depth: max recursion depth of JSON message to be deserialized.
  425. JSON messages over this depth will fail to be deserialized. Default value
  426. is 100.
  427. Returns:
  428. The same message passed as argument.
  429. """
  430. parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
  431. parser.ConvertMessage(js_dict, message, '')
  432. return message
  433. _INT_OR_FLOAT = (int, float)
  434. _LIST_LIKE = (list, tuple)
  435. class _Parser(object):
  436. """JSON format parser for protocol message."""
  437. def __init__(
  438. self, ignore_unknown_fields, descriptor_pool, max_recursion_depth
  439. ):
  440. self.ignore_unknown_fields = ignore_unknown_fields
  441. self.descriptor_pool = descriptor_pool
  442. self.max_recursion_depth = max_recursion_depth
  443. self.recursion_depth = 0
  444. def ConvertMessage(self, value, message, path):
  445. """Convert a JSON object into a message.
  446. Args:
  447. value: A JSON object.
  448. message: A WKT or regular protocol message to record the data.
  449. path: parent path to log parse error info.
  450. Raises:
  451. ParseError: In case of convert problems.
  452. """
  453. # Increment recursion depth at message entry. The max_recursion_depth limit
  454. # is exclusive: a depth value equal to max_recursion_depth will trigger an
  455. # error. For example, with max_recursion_depth=5, nesting up to depth 4 is
  456. # allowed, but attempting depth 5 raises ParseError.
  457. self.recursion_depth += 1
  458. if self.recursion_depth > self.max_recursion_depth:
  459. raise ParseError(
  460. 'Message too deep. Max recursion depth is {0}'.format(
  461. self.max_recursion_depth
  462. )
  463. )
  464. message_descriptor = message.DESCRIPTOR
  465. full_name = message_descriptor.full_name
  466. if not path:
  467. path = message_descriptor.name
  468. if _IsWrapperMessage(message_descriptor):
  469. self._ConvertWrapperMessage(value, message, path)
  470. elif full_name in _WKTJSONMETHODS:
  471. methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
  472. else:
  473. self._ConvertFieldValuePair(value, message, path)
  474. self.recursion_depth -= 1
  475. def _ConvertFieldValuePair(self, js, message, path):
  476. """Convert field value pairs into regular message.
  477. Args:
  478. js: A JSON object to convert the field value pairs.
  479. message: A regular protocol message to record the data.
  480. path: parent path to log parse error info.
  481. Raises:
  482. ParseError: In case of problems converting.
  483. """
  484. names = []
  485. message_descriptor = message.DESCRIPTOR
  486. fields_by_json_name = dict(
  487. (f.json_name, f) for f in message_descriptor.fields
  488. )
  489. def _ClearFieldOrExtension(message, field):
  490. if field.is_extension:
  491. message.ClearExtension(field)
  492. else:
  493. message.ClearField(field.name)
  494. def _GetFieldOrExtension(message, field):
  495. if field.is_extension:
  496. return message.Extensions[field]
  497. else:
  498. return getattr(message, field.name)
  499. def _SetFieldOrExtension(message, field, value):
  500. if field.is_extension:
  501. message.Extensions[field] = value
  502. else:
  503. setattr(message, field.name, value)
  504. for name in js:
  505. try:
  506. field = fields_by_json_name.get(name, None)
  507. if not field:
  508. field = message_descriptor.fields_by_name.get(name, None)
  509. if not field and _VALID_EXTENSION_NAME.match(name):
  510. if not message_descriptor.is_extendable:
  511. raise ParseError(
  512. 'Message type {0} does not have extensions at {1}'.format(
  513. message_descriptor.full_name, path
  514. )
  515. )
  516. identifier = name[1:-1] # strip [] brackets
  517. # pylint: disable=protected-access
  518. field = message.Extensions._FindExtensionByName(identifier)
  519. # pylint: enable=protected-access
  520. if not field:
  521. # Try looking for extension by the message type name, dropping the
  522. # field name following the final . separator in full_name.
  523. identifier = '.'.join(identifier.split('.')[:-1])
  524. # pylint: disable=protected-access
  525. field = message.Extensions._FindExtensionByName(identifier)
  526. # pylint: enable=protected-access
  527. if not field:
  528. if self.ignore_unknown_fields:
  529. continue
  530. raise ParseError(
  531. (
  532. 'Message type "{0}" has no field named "{1}" at "{2}".\n'
  533. ' Available Fields(except extensions): "{3}"'
  534. ).format(
  535. message_descriptor.full_name,
  536. name,
  537. path,
  538. [f.json_name for f in message_descriptor.fields],
  539. )
  540. )
  541. if name in names:
  542. raise ParseError(
  543. 'Message type "{0}" should not have multiple '
  544. '"{1}" fields at "{2}".'.format(
  545. message.DESCRIPTOR.full_name, name, path
  546. )
  547. )
  548. names.append(name)
  549. value = js[name]
  550. # Check no other oneof field is parsed.
  551. if field.containing_oneof is not None and value is not None:
  552. oneof_name = field.containing_oneof.name
  553. if oneof_name in names:
  554. raise ParseError(
  555. 'Message type "{0}" should not have multiple '
  556. '"{1}" oneof fields at "{2}".'.format(
  557. message.DESCRIPTOR.full_name, oneof_name, path
  558. )
  559. )
  560. names.append(oneof_name)
  561. if value is None:
  562. if (
  563. field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
  564. and field.message_type.full_name == 'google.protobuf.Value'
  565. ):
  566. sub_message = _GetFieldOrExtension(message, field)
  567. sub_message.null_value = 0
  568. elif (
  569. field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
  570. and field.enum_type.full_name == 'google.protobuf.NullValue'
  571. ):
  572. _SetFieldOrExtension(message, field, 0)
  573. else:
  574. _ClearFieldOrExtension(message, field)
  575. continue
  576. # Parse field value.
  577. if _IsMapEntry(field):
  578. _ClearFieldOrExtension(message, field)
  579. self._ConvertMapFieldValue(
  580. value, message, field, '{0}.{1}'.format(path, name)
  581. )
  582. elif field.is_repeated:
  583. _ClearFieldOrExtension(message, field)
  584. if not isinstance(value, _LIST_LIKE):
  585. raise ParseError(
  586. 'repeated field {0} must be in [] which is {1} at {2}'.format(
  587. name, value, path
  588. )
  589. )
  590. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  591. # Repeated message field.
  592. for index, item in enumerate(value):
  593. sub_message = _GetFieldOrExtension(message, field).add()
  594. # None is a null_value in Value.
  595. if (
  596. item is None
  597. and sub_message.DESCRIPTOR.full_name
  598. != 'google.protobuf.Value'
  599. ):
  600. raise ParseError(
  601. 'null is not allowed to be used as an element'
  602. ' in a repeated field at {0}.{1}[{2}]'.format(
  603. path, name, index
  604. )
  605. )
  606. self.ConvertMessage(
  607. item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)
  608. )
  609. else:
  610. # Repeated scalar field.
  611. for index, item in enumerate(value):
  612. if item is None:
  613. raise ParseError(
  614. 'null is not allowed to be used as an element'
  615. ' in a repeated field at {0}.{1}[{2}]'.format(
  616. path, name, index
  617. )
  618. )
  619. self._ConvertAndAppendScalar(
  620. message, field, item, '{0}.{1}[{2}]'.format(path, name, index)
  621. )
  622. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  623. sub_message = _GetFieldOrExtension(message, field)
  624. sub_message.SetInParent()
  625. self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
  626. else:
  627. self._ConvertAndSetScalar(
  628. message, field, value, '{0}.{1}'.format(path, name)
  629. )
  630. except ParseError as e:
  631. if field and field.containing_oneof is None:
  632. raise ParseError(
  633. 'Failed to parse {0} field: {1}.'.format(name, e)
  634. ) from e
  635. else:
  636. raise ParseError(str(e)) from e
  637. except ValueError as e:
  638. raise ParseError(
  639. 'Failed to parse {0} field: {1}.'.format(name, e)
  640. ) from e
  641. except TypeError as e:
  642. raise ParseError(
  643. 'Failed to parse {0} field: {1}.'.format(name, e)
  644. ) from e
  645. def _ConvertAnyMessage(self, value, message, path):
  646. """Convert a JSON representation into Any message."""
  647. if isinstance(value, dict) and not value:
  648. return
  649. try:
  650. type_url = value['@type']
  651. except KeyError as e:
  652. raise ParseError(
  653. '@type is missing when parsing any message at {0}'.format(path)
  654. ) from e
  655. try:
  656. sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
  657. except TypeError as e:
  658. raise ParseError('{0} at {1}'.format(e, path)) from e
  659. message_descriptor = sub_message.DESCRIPTOR
  660. full_name = message_descriptor.full_name
  661. if _IsWrapperMessage(message_descriptor):
  662. self._ConvertWrapperMessage(
  663. value['value'], sub_message, '{0}.value'.format(path)
  664. )
  665. elif full_name in _WKTJSONMETHODS:
  666. # For well-known types (including nested Any), use ConvertMessage
  667. # to ensure recursion depth is properly tracked
  668. self.ConvertMessage(value['value'], sub_message, '{0}.value'.format(path))
  669. else:
  670. del value['@type']
  671. try:
  672. self._ConvertFieldValuePair(value, sub_message, path)
  673. finally:
  674. value['@type'] = type_url
  675. # Sets Any message
  676. message.value = sub_message.SerializeToString()
  677. message.type_url = type_url
  678. def _ConvertGenericMessage(self, value, message, path):
  679. """Convert a JSON representation into message with FromJsonString."""
  680. # Duration, Timestamp, FieldMask have a FromJsonString method to do the
  681. # conversion. Users can also call the method directly.
  682. try:
  683. message.FromJsonString(value)
  684. except ValueError as e:
  685. raise ParseError('{0} at {1}'.format(e, path)) from e
  686. def _ConvertValueMessage(self, value, message, path):
  687. """Convert a JSON representation into Value message."""
  688. if isinstance(value, dict):
  689. self.ConvertMessage(value, message.struct_value, path)
  690. elif isinstance(value, _LIST_LIKE):
  691. self.ConvertMessage(value, message.list_value, path)
  692. elif value is None:
  693. message.null_value = 0
  694. elif isinstance(value, bool):
  695. message.bool_value = value
  696. elif isinstance(value, str):
  697. message.string_value = value
  698. elif isinstance(value, _INT_OR_FLOAT):
  699. message.number_value = value
  700. else:
  701. raise ParseError(
  702. 'Value {0} has unexpected type {1} at {2}'.format(
  703. value, type(value), path
  704. )
  705. )
  706. def _ConvertListOrTupleValueMessage(self, value, message, path):
  707. """Convert a JSON representation into ListValue message."""
  708. if not isinstance(value, _LIST_LIKE):
  709. raise ParseError(
  710. 'ListValue must be in [] which is {0} at {1}'.format(value, path)
  711. )
  712. message.ClearField('values')
  713. for index, item in enumerate(value):
  714. self.ConvertMessage(
  715. item, message.values.add(), '{0}[{1}]'.format(path, index)
  716. )
  717. def _ConvertStructMessage(self, value, message, path):
  718. """Convert a JSON representation into Struct message."""
  719. if not isinstance(value, dict):
  720. raise ParseError(
  721. 'Struct must be in a dict which is {0} at {1}'.format(value, path)
  722. )
  723. # Clear will mark the struct as modified so it will be created even if
  724. # there are no values.
  725. message.Clear()
  726. for key in value:
  727. self.ConvertMessage(
  728. value[key], message.fields[key], '{0}.{1}'.format(path, key)
  729. )
  730. return
  731. def _ConvertWrapperMessage(self, value, message, path):
  732. """Convert a JSON representation into Wrapper message."""
  733. field = message.DESCRIPTOR.fields_by_name['value']
  734. self._ConvertAndSetScalar(
  735. message, field, value, path='{0}.value'.format(path)
  736. )
  737. def _ConvertMapFieldValue(self, value, message, field, path):
  738. """Convert map field value for a message map field.
  739. Args:
  740. value: A JSON object to convert the map field value.
  741. message: A protocol message to record the converted data.
  742. field: The descriptor of the map field to be converted.
  743. path: parent path to log parse error info.
  744. Raises:
  745. ParseError: In case of convert problems.
  746. """
  747. if not isinstance(value, dict):
  748. raise ParseError(
  749. 'Map field {0} must be in a dict which is {1} at {2}'.format(
  750. field.name, value, path
  751. )
  752. )
  753. key_field = field.message_type.fields_by_name['key']
  754. value_field = field.message_type.fields_by_name['value']
  755. for key in value:
  756. key_value = _ConvertScalarFieldValue(
  757. key, key_field, '{0}.key'.format(path), True
  758. )
  759. if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
  760. self.ConvertMessage(
  761. value[key],
  762. getattr(message, field.name)[key_value],
  763. '{0}[{1}]'.format(path, key_value),
  764. )
  765. else:
  766. self._ConvertAndSetScalarToMapKey(
  767. message,
  768. field,
  769. key_value,
  770. value[key],
  771. path='{0}[{1}]'.format(path, key_value),
  772. )
  773. def _ConvertAndSetScalar(self, message, field, js_value, path):
  774. """Convert scalar from js_value and assign it to message.field."""
  775. try:
  776. value = _ConvertScalarFieldValue(js_value, field, path)
  777. if field.is_extension:
  778. message.Extensions[field] = value
  779. else:
  780. setattr(message, field.name, value)
  781. except EnumStringValueParseError:
  782. if not self.ignore_unknown_fields:
  783. raise
  784. def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path):
  785. """Convert scalar from js_value and append it to message.repeated_field."""
  786. try:
  787. if repeated_field.is_extension:
  788. repeated = message.Extensions[repeated_field]
  789. else:
  790. repeated = getattr(message, repeated_field.name)
  791. value = _ConvertScalarFieldValue(js_value, repeated_field, path)
  792. repeated.append(value)
  793. except EnumStringValueParseError:
  794. if not self.ignore_unknown_fields:
  795. raise
  796. def _ConvertAndSetScalarToMapKey(
  797. self, message, map_field, converted_key, js_value, path
  798. ):
  799. """Convert scalar from 'js_value' and add it to message.map_field[converted_key]."""
  800. try:
  801. getattr(message, map_field.name)[converted_key] = (
  802. _ConvertScalarFieldValue(
  803. js_value,
  804. map_field.message_type.fields_by_name['value'],
  805. path,
  806. )
  807. )
  808. except EnumStringValueParseError:
  809. if not self.ignore_unknown_fields:
  810. raise
  811. def _GetJsonEnumValueOption(ev):
  812. """Helper to get the JsonEnumValueOptions for an enum value.
  813. Args:
  814. ev: The EnumValueDescriptor.
  815. Returns:
  816. The JsonEnumValueOptions message if the extension is present,
  817. otherwise None.
  818. """
  819. try:
  820. extension_descriptor = descriptor_pool.Default().FindExtensionByName(
  821. 'pb.enumvalue.json'
  822. )
  823. except KeyError:
  824. return None
  825. if ev.GetOptions().HasExtension(extension_descriptor):
  826. return ev.GetOptions().Extensions[extension_descriptor]
  827. return None
  828. def _ConvertScalarFieldValue(value, field, path, require_str=False):
  829. """Convert a single scalar field value.
  830. Args:
  831. value: A scalar value to convert the scalar field value.
  832. field: The descriptor of the field to convert.
  833. path: parent path to log parse error info.
  834. require_str: If True, the field value must be a str.
  835. Returns:
  836. The converted scalar field value
  837. Raises:
  838. ParseError: In case of convert problems.
  839. EnumStringValueParseError: In case of unknown enum string value.
  840. """
  841. try:
  842. if field.cpp_type in _INT_TYPES:
  843. return _ConvertInteger(value)
  844. elif field.cpp_type in _FLOAT_TYPES:
  845. return _ConvertFloat(value, field)
  846. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
  847. return _ConvertBool(value, require_str)
  848. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
  849. if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
  850. if isinstance(value, str):
  851. encoded = value.encode('utf-8')
  852. else:
  853. encoded = value
  854. # Add extra padding '='
  855. padded_value = encoded + b'=' * (4 - len(encoded) % 4)
  856. return base64.urlsafe_b64decode(padded_value)
  857. else:
  858. # Checking for unpaired surrogates appears to be unreliable,
  859. # depending on the specific Python version, so we check manually.
  860. if _UNPAIRED_SURROGATE_PATTERN.search(value):
  861. raise ParseError('Unpaired surrogate')
  862. return value
  863. elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
  864. # Convert an enum value.
  865. enum_value = field.enum_type.values_by_name.get(value, None)
  866. # First check to see if we have a custom enum string.
  867. if enum_value is None:
  868. for ev in field.enum_type.values:
  869. option = _GetJsonEnumValueOption(ev)
  870. if option is not None and option.string == value:
  871. enum_value = ev
  872. break
  873. # If not, try parsing it as an integer.
  874. if enum_value is None:
  875. try:
  876. number = int(value)
  877. enum_value = field.enum_type.values_by_number.get(number, None)
  878. except ValueError as e:
  879. # Since parsing to integer failed and lookup in values_by_name didn't
  880. # find this name, we have an enum string value which is unknown.
  881. raise EnumStringValueParseError(
  882. 'Invalid enum value {0} for enum type {1}'.format(
  883. value, field.enum_type.full_name
  884. )
  885. ) from e
  886. if enum_value is None:
  887. if field.enum_type.is_closed:
  888. raise ParseError(
  889. 'Invalid enum value {0} for enum type {1}'.format(
  890. value, field.enum_type.full_name
  891. )
  892. )
  893. else:
  894. return number
  895. return enum_value.number
  896. except EnumStringValueParseError as e:
  897. raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e
  898. except ParseError as e:
  899. raise ParseError('{0} at {1}'.format(e, path)) from e
  900. def _ConvertInteger(value):
  901. """Convert an integer.
  902. Args:
  903. value: A scalar value to convert.
  904. Returns:
  905. The integer value.
  906. Raises:
  907. ParseError: If an integer couldn't be consumed.
  908. """
  909. if isinstance(value, float) and not value.is_integer():
  910. raise ParseError("Couldn't parse integer: {0}".format(value))
  911. if isinstance(value, str) and value.find(' ') != -1:
  912. raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
  913. if isinstance(value, bool):
  914. raise ParseError(
  915. 'Bool value {0} is not acceptable for integer field'.format(value)
  916. )
  917. try:
  918. return int(value)
  919. except ValueError as e:
  920. # Attempt to parse as an integer-valued float.
  921. try:
  922. f = float(value)
  923. except ValueError:
  924. # Raise the original exception for the int parse.
  925. raise e # pylint: disable=raise-missing-from
  926. if not f.is_integer():
  927. raise ParseError(
  928. 'Couldn\'t parse non-integer string: "{0}"'.format(value)
  929. ) from e
  930. return int(f)
  931. def _ConvertFloat(value, field):
  932. """Convert an floating point number."""
  933. if isinstance(value, float):
  934. if math.isnan(value):
  935. raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
  936. if math.isinf(value):
  937. if value > 0:
  938. raise ParseError(
  939. "Couldn't parse Infinity or value too large, "
  940. 'use quoted "Infinity" instead'
  941. )
  942. else:
  943. raise ParseError(
  944. "Couldn't parse -Infinity or value too small, "
  945. 'use quoted "-Infinity" instead'
  946. )
  947. if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
  948. # pylint: disable=protected-access
  949. if value > type_checkers._FLOAT_MAX:
  950. raise ParseError('Float value too large')
  951. # pylint: disable=protected-access
  952. if value < type_checkers._FLOAT_MIN:
  953. raise ParseError('Float value too small')
  954. if value == 'nan':
  955. raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
  956. try:
  957. # Assume Python compatible syntax.
  958. return float(value)
  959. except ValueError as e:
  960. # Check alternative spellings.
  961. if value == _NEG_INFINITY:
  962. return float('-inf')
  963. elif value == _INFINITY:
  964. return float('inf')
  965. elif value == _NAN:
  966. return float('nan')
  967. else:
  968. raise ParseError("Couldn't parse float: {0}".format(value)) from e
  969. def _ConvertBool(value, require_str):
  970. """Convert a boolean value.
  971. Args:
  972. value: A scalar value to convert.
  973. require_str: If True, value must be a str.
  974. Returns:
  975. The bool parsed.
  976. Raises:
  977. ParseError: If a boolean value couldn't be consumed.
  978. """
  979. if require_str:
  980. if value == 'true':
  981. return True
  982. elif value == 'false':
  983. return False
  984. else:
  985. raise ParseError('Expected "true" or "false", not {0}'.format(value))
  986. if not isinstance(value, bool):
  987. raise ParseError('Expected true or false without quotes')
  988. return value
  989. _WKTJSONMETHODS = {
  990. 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],
  991. 'google.protobuf.Duration': [
  992. '_GenericMessageToJsonObject',
  993. '_ConvertGenericMessage',
  994. ],
  995. 'google.protobuf.FieldMask': [
  996. '_GenericMessageToJsonObject',
  997. '_ConvertGenericMessage',
  998. ],
  999. 'google.protobuf.ListValue': [
  1000. '_ListValueMessageToJsonObject',
  1001. '_ConvertListOrTupleValueMessage',
  1002. ],
  1003. 'google.protobuf.Struct': [
  1004. '_StructMessageToJsonObject',
  1005. '_ConvertStructMessage',
  1006. ],
  1007. 'google.protobuf.Timestamp': [
  1008. '_GenericMessageToJsonObject',
  1009. '_ConvertGenericMessage',
  1010. ],
  1011. 'google.protobuf.Value': [
  1012. '_ValueMessageToJsonObject',
  1013. '_ConvertValueMessage',
  1014. ],
  1015. }