diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index a901a72692..6552d2dbb8 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -78,6 +78,19 @@ def self.lex(src, filename = "-", lineno = 1, raise_errors: false) end end + # Tokenizes the Ruby program and returns an array of strings. + # The +filename+ and +lineno+ arguments are mostly ignored, since the + # return value is just the tokenized input. + # By default, this method does not handle syntax errors in +src+, + # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+. + # + # p Ripper.tokenize("def m(a) nil end") + # # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"] + # + def self.tokenize(...) + lex(...).map(&:value) + end + # This contains a table of all of the parser events and their # corresponding arity. PARSER_EVENT_TABLE = { diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 2bd9c2fe4a..cac20a073d 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -84,6 +84,11 @@ class RipperTest < TestCase define_method("#{fixture.test_name}_lexer_parse") { assert_ripper_lexer_parse(fixture.read) } end + def test_tokenize + source = "foo;1;BAZ" + assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source)) + end + # Check that the hardcoded values don't change without us noticing. def test_internals actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort