Skip to content
Extraits de code Groupes Projets
Vérifiée Valider 5d3af0ea rédigé par Kubat's avatar Kubat
Parcourir les fichiers

LEXER: Parse integers before non-alpha-numeric simple tokens for negative numbers

parent 5b19f324
Aucune branche associée trouvée
Aucune étiquette associée trouvée
1 requête de fusion!25Draft: New Vivy module spec
......@@ -85,55 +85,55 @@ tokenizeFile(const char *file, TokenList *tokens, std::string *storage)
for (;;) {
global_continue:
/* First simple tokens that are not alpha-numeric */
/* Find a floating? */
fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount);
for (const auto &simpleToken : Vivy::Script::SIMPLE_TOKENS_NON_ALPHANUM) {
if (StrV::startsWith(fileContent, simpleToken)) {
/* If the charatecr is an UTF8 character, the Location can skip
* multiple chars because the glyph is multiple characters long! */
tokens->push_back(Token::fromSimple(loc, simpleToken));
fileContent.chopLeft(simpleToken.len());
loc = Location::shift(loc, simpleToken.len());
goto global_continue;
if (fileContent.tryChopFloating(&floating, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid floating point found, directly "
"followed by an identifier...");
}
tokens->push_back(Token::fromFloating(loc, floating));
loc = Location::shift(loc, trimmedAmount);
continue;
}
/* Find a string literal? */
/* Find an integer? */
fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopEscapedString(&qualifiedName, &trimmedAmount)) {
if (fileContent.tryChopInteger(&integer, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid string literal, directly "
throw std::runtime_error("Invalid integer found, directly "
"followed by an identifier...");
}
tokens->push_back(Token::fromStringLit(loc, qualifiedName));
tokens->push_back(Token::fromInteger(loc, integer));
loc = Location::shift(loc, trimmedAmount);
continue;
}
/* Find a floating? */
/* First simple tokens that are not alpha-numeric */
fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopFloating(&floating, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid floating point found, directly "
"followed by an identifier...");
for (const auto &simpleToken : Vivy::Script::SIMPLE_TOKENS_NON_ALPHANUM) {
if (StrV::startsWith(fileContent, simpleToken)) {
/* If the charatecr is an UTF8 character, the Location can skip
* multiple chars because the glyph is multiple characters long! */
tokens->push_back(Token::fromSimple(loc, simpleToken));
fileContent.chopLeft(simpleToken.len());
loc = Location::shift(loc, simpleToken.len());
goto global_continue;
}
tokens->push_back(Token::fromFloating(loc, floating));
loc = Location::shift(loc, trimmedAmount);
continue;
}
/* Find an integer? */
/* Find a string literal? */
fileContent = StrV::trimL(fileContent, &trimmedAmount);
loc = Location::shift(loc, trimmedAmount);
if (fileContent.tryChopInteger(&integer, &trimmedAmount)) {
if (fileContent.tryChopEscapedString(&qualifiedName, &trimmedAmount)) {
if (fileContent.chopNextIdChar(&ok); ok) {
throw std::runtime_error("Invalid integer found, directly "
throw std::runtime_error("Invalid string literal, directly "
"followed by an identifier...");
}
tokens->push_back(Token::fromInteger(loc, integer));
tokens->push_back(Token::fromStringLit(loc, qualifiedName));
loc = Location::shift(loc, trimmedAmount);
continue;
}
......
0% Chargement en cours ou .
You are about to add 0 people to the discussion. Proceed with caution.
Veuillez vous inscrire ou vous pour commenter