7 examples of 'python string tokenizer' in Python

13
def tokenize(st):
14
    return " ".join([str(x) for x in tokenizer(st)])

20
@pytest.mark.parametrize(
21
    "text,expected",
22
    [
23
        ("Hello %s", ["Hello ", "%s"]),
24
        ("Hello %(username)s", ["Hello ", "%(username)s"]),
25
        ("Hello %(user)s%(name)s", ["Hello ", "%(user)s", "%(name)s"]),
26
        ("Hello {username}", ["Hello ", "{username}"]),
27
        ("Hello {user}{name}", ["Hello ", "{user}", "{name}"]),
28
        ("Products and Services", ["Products and Services"]),
29
    ],
30
)
31
def test_python_tokenizing(text, expected):
32
    vartok = VariableTokenizer(["python-format", "python-brace-format"])
33
    assert vartok.tokenize(text) == expected

487
def _default_tokenizer(s):
488
  """Default string tokenizer which splits on newlines."""
489
  return s.split('\n')

210
def __init__(self, string):
211
    """Prepare to tokenize the provided code.
212
213
    :param str string: The source code, as a string.
214
    """
215
    assert "\t" not in string, (
216
        "Remove tabs from code before attempting to tokenize. "
217
        "We don't provide meaningful token positions for code "
218
        "that has tabs in it."
219
    )
220
221
    # Add a dummy whitespace character to end the last token.
222
    self._string = string + "\n"
223
224
    # The 'cursor' is the index into the source code string where we
225
    # currently are. We advance this as we consume tokens.
226
    self._cursor = 0
227
228
    # The row and column corresponding to the cursor. When we hit a newline
229
    # character, the row increments and the column is set to zero.
230
    self._row = 0
231
    self._column = 0

161
def __init__(self, string):
162
    """Tokenize the provided code."""
163
    # Add a dummy whitespace character to end the last token.
164
    self._string = string + "\n"
165
    self._cursor = 0
166
167
    self._row = 0
168
    self._column = 0

67
def tokenize(self, input):
68
    self.rv = []
69
    self.lineno = 1
70
    super(ASDLScanner, self).tokenize(input)
71
    return self.rv

180
def split(self, text):
181
    return text.split()