google / google/fonts

Unicode errors reading various METADATA.pb files with Python 3

Open
#1,591 8 comments 0 reactions 0 assignees View on GitHub
I API / Website / Platform I Tools / workflow / repo
Dominant language
HTML
Stars
20.5k
Forks
2.9k
Avg merge
1d 21h
Merged PRs (30d)
95

Description

Can anyone help me understand how I can get around this issue please? I want to read the METADATA.pb of all the Google fonts included in the zipfile https://github.com/google/fonts/archive/master.zip.

The code that reads all the METADATA.pb files after unzipping the Google fonts zipfile:
```
import os
from fonts_public_pb2 import FontProto, FamilyProto
from google.protobuf import text_format

for dirpath, dirnames, filenames in os.walk('/usr/share/fonts/truetype'):
metadataProtoFile = os.path.join(dirpath, 'METADATA.pb')
if not os.path.exists(metadataProtoFile):
continue

metadata = FamilyProto()
with open(metadataProtoFile) as f:
text_data = f.read()
try:
text_format.Merge(text_data, metadata)
except Exception as e:
print(metadataProtoFile)
print(repr(e))
continue
```

The errors:
```
(venv35) ubuntu@ip-10-0-0-196:~/code/fontprocessor$ python tmp.py
/usr/share/fonts/truetype/imfellgreatprimer/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/ptsansnarrow/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/boogaloo/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 17: invalid continuation byte",)
/usr/share/fonts/truetype/nanumbrushscript/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/imfellenglish/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/imfellfrenchcanon/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/playfairdisplaysc/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xf8 in position 14: invalid start byte",)
/usr/share/fonts/truetype/londrinashadow/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe3 in position 14: invalid continuation byte",)
/usr/share/fonts/truetype/alfaslabone/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 6: unexpected end of data",)
/usr/share/fonts/truetype/chauphilomeneone/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xf3 in position 11: invalid continuation byte",)
/usr/share/fonts/truetype/ptsans/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/imfelldoublepicasc/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/ptserif/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/ptsanscaption/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/sourceserifpro/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xdf in position 10: invalid continuation byte",)
/usr/share/fonts/truetype/bitter/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/macondoswashcaps/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 17: invalid continuation byte",)
/usr/share/fonts/truetype/nanumgothic/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/esteban/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 3: invalid continuation byte",)
/usr/share/fonts/truetype/buda/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe8 in position 2: invalid continuation byte",)
/usr/share/fonts/truetype/imfellenglishsc/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/coiny/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe3 in position 14: invalid continuation byte",)
/usr/share/fonts/truetype/imfellgreatprimersc/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/andadasc/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/pollerone/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xfc in position 10: invalid start byte",)
/usr/share/fonts/truetype/kronaone/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xfc in position 10: invalid start byte",)
/usr/share/fonts/truetype/cambo/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/signikanegative/METADATA.pb
IndexError('list index out of range',)
/usr/share/fonts/truetype/durusans/METADATA.pb
IndexError('list index out of range',)
/usr/share/fonts/truetype/andada/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/imfellfrenchcanonsc/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/playfairdisplay/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xf8 in position 14: invalid start byte",)
/usr/share/fonts/truetype/martelsans/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 23: invalid continuation byte",)
/usr/share/fonts/truetype/voltaire/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xfc in position 10: invalid start byte",)
/usr/share/fonts/truetype/ptserifcaption/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/economica/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xf3 in position 11: invalid continuation byte",)
/usr/share/fonts/truetype/londrinaoutline/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe3 in position 14: invalid continuation byte",)
/usr/share/fonts/truetype/ptmono/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/telex/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/nanumgothiccoding/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/fenix/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xed in position 10: invalid continuation byte",)
/usr/share/fonts/truetype/germaniaone/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 17: invalid continuation byte",)
/usr/share/fonts/truetype/nanummyeongjo/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/tauri/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xfc in position 10: invalid start byte",)
/usr/share/fonts/truetype/sumana/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/poly/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 5: invalid continuation byte",)
/usr/share/fonts/truetype/macondo/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 17: invalid continuation byte",)
/usr/share/fonts/truetype/gochihand/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/nanumpenscript/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 10: invalid start byte",)
/usr/share/fonts/truetype/londrinasolid/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe3 in position 14: invalid continuation byte",)
/usr/share/fonts/truetype/inknutantiqua/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xf8 in position 14: invalid start byte",)
/usr/share/fonts/truetype/imfelldwpicasc/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/signika/METADATA.pb
IndexError('list index out of range',)
/usr/share/fonts/truetype/imfelldoublepica/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/imfelldwpica/METADATA.pb
ParseError("14:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xa9 in position 0: invalid start byte",)
/usr/share/fonts/truetype/acme/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 13: invalid continuation byte",)
/usr/share/fonts/truetype/noticiatext/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 6: unexpected end of data",)
/usr/share/fonts/truetype/salsa/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe1 in position 17: invalid continuation byte",)
/usr/share/fonts/truetype/lato/METADATA.pb
IndexError('list index out of range',)
/usr/share/fonts/truetype/biryani/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 23: invalid continuation byte",)
/usr/share/fonts/truetype/londrinasketch/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe3 in position 14: invalid continuation byte",)
/usr/share/fonts/truetype/carme/METADATA.pb
ParseError("3:1 : Couldn't parse string: 'utf-8' codec can't decode byte 0xe9 in position 3: invalid continuation byte",)
(venv35) ubuntu@ip-10-0-0-196:~/code/fontprocessor$
```

fonts_public_pb2.py - nabbed from FontBakery
```
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: fonts_public.proto

import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()

DESCRIPTOR = _descriptor.FileDescriptor(
name='fonts_public.proto',
package='google.fonts',
serialized_pb=_b('\n\x12\x66onts_public.proto\x12\x0cgoogle.fonts\"\xae\x01\n\x0b\x46\x61milyProto\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\x10\n\x08\x64\x65signer\x18\x02 \x02(\t\x12\x0f\n\x07license\x18\x03 \x02(\t\x12\x10\n\x08\x63\x61tegory\x18\x04 \x02(\t\x12\x12\n\ndate_added\x18\x05 \x02(\t\x12&\n\x05\x66onts\x18\x06 \x03(\x0b\x32\x17.google.fonts.FontProto\x12\x0f\n\x07\x61liases\x18\x07 \x03(\t\x12\x0f\n\x07subsets\x18\x08 \x03(\t\"\x8a\x01\n\tFontProto\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05style\x18\x02 \x02(\t\x12\x0e\n\x06weight\x18\x03 \x02(\x05\x12\x10\n\x08\x66ilename\x18\x04 \x02(\t\x12\x18\n\x10post_script_name\x18\x05 \x02(\t\x12\x11\n\tfull_name\x18\x06 \x02(\t\x12\x11\n\tcopyright\x18\x07 \x01(\tB%\n\x16\x63om.google.fonts.protoB\x0b\x46ontsPublic')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)

_FAMILYPROTO = _descriptor.Descriptor(
name='FamilyProto',
full_name='google.fonts.FamilyProto',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='google.fonts.FamilyProto.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='designer', full_name='google.fonts.FamilyProto.designer', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='license', full_name='google.fonts.FamilyProto.license', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='category', full_name='google.fonts.FamilyProto.category', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='date_added', full_name='google.fonts.FamilyProto.date_added', index=4,
number=5, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='fonts', full_name='google.fonts.FamilyProto.fonts', index=5,
number=6, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='aliases', full_name='google.fonts.FamilyProto.aliases', index=6,
number=7, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='subsets', full_name='google.fonts.FamilyProto.subsets', index=7,
number=8, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=37,
serialized_end=211,
)

_FONTPROTO = _descriptor.Descriptor(
name='FontProto',
full_name='google.fonts.FontProto',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='google.fonts.FontProto.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='style', full_name='google.fonts.FontProto.style', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='weight', full_name='google.fonts.FontProto.weight', index=2,
number=3, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='filename', full_name='google.fonts.FontProto.filename', index=3,
number=4, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='post_script_name', full_name='google.fonts.FontProto.post_script_name', index=4,
number=5, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='full_name', full_name='google.fonts.FontProto.full_name', index=5,
number=6, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='copyright', full_name='google.fonts.FontProto.copyright', index=6,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
oneofs=[
],
serialized_start=214,
serialized_end=352,
)

_FAMILYPROTO.fields_by_name['fonts'].message_type = _FONTPROTO
DESCRIPTOR.message_types_by_name['FamilyProto'] = _FAMILYPROTO
DESCRIPTOR.message_types_by_name['FontProto'] = _FONTPROTO

FamilyProto = _reflection.GeneratedProtocolMessageType('FamilyProto', (_message.Message,), dict(
DESCRIPTOR = _FAMILYPROTO,
__module__ = 'fonts_public_pb2'
# @@protoc_insertion_point(class_scope:google.fonts.FamilyProto)
))
_sym_db.RegisterMessage(FamilyProto)

FontProto = _reflection.GeneratedProtocolMessageType('FontProto', (_message.Message,), dict(
DESCRIPTOR = _FONTPROTO,
__module__ = 'fonts_public_pb2'
# @@protoc_insertion_point(class_scope:google.fonts.FontProto)
))
_sym_db.RegisterMessage(FontProto)

DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.fonts.protoB\013FontsPublic'))
# @@protoc_insertion_point(module_scope)
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.