From 9a979a0ddfcc2e8dd14605a2fb7bc1f819eab4d3 Mon Sep 17 00:00:00 2001 From: Jun-ya Norimatsu Date: Fri, 17 Jul 2020 22:34:49 +0900 Subject: [PATCH] ensure str for the case of bytes type. --- create_pretraining_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_pretraining_data.py b/create_pretraining_data.py index 8ae02a0f..5e882d16 100644 --- a/create_pretraining_data.py +++ b/create_pretraining_data.py @@ -402,7 +402,7 @@ def _is_start_piece_sp(piece): if (six.ensure_str(piece).startswith("▁") or six.ensure_str(piece).startswith("<") or piece in special_pieces or not all([i.lower() in english_chars.union(special_pieces) - for i in piece])): + for i in six.ensure_str(piece)])): return True else: return False