Skip to content
Extraits de code Groupes Projets
Vérifiée Valider 5d3af0ea rédigé par Kubat's avatar Kubat
Parcourir les fichiers

LEXER: Parse integers before non-alpha-numeric simple tokens for negative numbers

parent 5b19f324
Aucune branche associée trouvée
Aucune étiquette associée trouvée
1 requête de fusion!25Draft: New Vivy module spec
...@@ -85,55 +85,55 @@ tokenizeFile(const char *file, TokenList *tokens, std::string *storage) ...@@ -85,55 +85,55 @@ tokenizeFile(const char *file, TokenList *tokens, std::string *storage)
for (;;) { for (;;) {
global_continue: global_continue:
/* First simple tokens that are not alpha-numeric */ /* Find a floating? */
fileContent = StrV::trimL(fileContent, &trimmedAmount); fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
for (const auto &simpleToken : Vivy::Script::SIMPLE_TOKENS_NON_ALPHANUM) { if (fileContent.tryChopFloating(&floating, &trimmedAmount)) {
if (StrV::startsWith(fileContent, simpleToken)) { if (fileContent.chopNextIdChar(&ok); ok) {
/* If the charatecr is an UTF8 character, the Location can skip throw std::runtime_error("Invalid floating point found, directly "
* multiple chars because the glyph is multiple characters long! */ "followed by an identifier...");
tokens->push_back(Token::fromSimple(loc, simpleToken));
fileContent.chopLeft(simpleToken.len());
loc = Location::shift(loc, simpleToken.len());
goto global_continue;
} }
tokens->push_back(Token::fromFloating(loc, floating));
loc = Location::shift(loc, trimmedAmount);
continue;
} }
/* Find a string literal? */ /* Find an integer? */
fileContent = StrV::trimL(fileContent, &trimmedAmount); fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopEscapedString(&qualifiedName, &trimmedAmount)) { if (fileContent.tryChopInteger(&integer, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) { if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid string literal, directly " throw std::runtime_error("Invalid integer found, directly "
"followed by an identifier..."); "followed by an identifier...");
} }
tokens->push_back(Token::fromStringLit(loc, qualifiedName)); tokens->push_back(Token::fromInteger(loc, integer));
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
continue; continue;
} }
/* Find a floating? */ /* First simple tokens that are not alpha-numeric */
fileContent = StrV::trimL(fileContent, &trimmedAmount); fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopFloating(&floating, &trimmedAmount)) { for (const auto &simpleToken : Vivy::Script::SIMPLE_TOKENS_NON_ALPHANUM) {
if (fileContent.chopNextIdChar(&ok); ok) { if (StrV::startsWith(fileContent, simpleToken)) {
throw std::runtime_error("Invalid floating point found, directly " /* If the charatecr is an UTF8 character, the Location can skip
"followed by an identifier..."); * multiple chars because the glyph is multiple characters long! */
tokens->push_back(Token::fromSimple(loc, simpleToken));
fileContent.chopLeft(simpleToken.len());
loc = Location::shift(loc, simpleToken.len());
goto global_continue;
} }
tokens->push_back(Token::fromFloating(loc, floating));
loc = Location::shift(loc, trimmedAmount);
continue;
} }
/* Find an integer? */ /* Find a string literal? */
fileContent = StrV::trimL(fileContent, &trimmedAmount); fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopInteger(&integer, &trimmedAmount)) { if (fileContent.tryChopEscapedString(&qualifiedName, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) { if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid integer found, directly " throw std::runtime_error("Invalid string literal, directly "
"followed by an identifier..."); "followed by an identifier...");
} }
tokens->push_back(Token::fromInteger(loc, integer)); tokens->push_back(Token::fromStringLit(loc, qualifiedName));
loc = Location::shift(loc, trimmedAmount); loc = Location::shift(loc, trimmedAmount);
continue; continue;
} }
......
0% Chargement en cours ou .
You are about to add 0 people to the discussion. Proceed with caution.
Veuillez vous inscrire ou vous pour commenter