Embedded/pikapython
1
0
Fork 0
mirror of https://gitee.com/Lyon1998/pikapython.git synced 2025-01-15 17:02:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

support multi module import parse

Browse Source
This commit is contained in:
pikastech 2022-10-20 12:55:07 +08:00
parent 2b117873d2
commit 7c6267bfd9
9 changed files with 360 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
port/linux/.vscode/launch.json vendored
View File

@ -11,7 +11,7 @@
"program": "${workspaceFolder}/build/test/pikascript_test",
// "program": "${workspaceFolder}/build/boot/demo06-pikamain/pikascript_demo06-pikamain",
"args": [
// "--gtest_filter=vm.c_module_get_set_attr"
"--gtest_filter=parser.multi_from_import_as"
],
"stopAtEntry": false,
"cwd": "${workspaceFolder}",

223
src/PikaParser.c
View File

@ -35,22 +35,28 @@
/* local head */
typedef QueueObj AST;
char* Lexer_getTokens(Args* outBuffs, char* stmt);
char* AST_genAsm(AST* ast, Args* outBuffs);
char* Lexer_parseLine(Args* outBuffs, char* stmt);
int32_t AST_deinit(AST* ast);
char* Parser_linesToAsm(Args* outBuffs, char* multiLine);
uint8_t Parser_isContainToken(char* tokens,
enum TokenType token_type,
char* pyload);
void Cursor_init(struct Cursor* cs);
void Cursor_parse(struct Cursor* cs, char* stmt);
void Cursor_deinit(struct Cursor* cs);
void Cursor_beforeIter(struct Cursor* cs);
uint8_t Tokens_isContain(char* tokens, enum TokenType token_type, char* pyload);
char* Tokens_pop(Args* buffs_p, char** tokens);
/* Cursor preivilage */
void _Cursor_init(struct Cursor* cs);
void _Cursor_parse(struct Cursor* cs, char* stmt);
void _Cursor_beforeIter(struct Cursor* cs);
/* Cursor iter api */
void Cursor_iterStart(struct Cursor* cs);
void Cursor_iterEnd(struct Cursor* cs);
char* Cursor_popToken(Args* buffs, char** tokens, char* devide);
char* Parser_popToken(Args* buffs_p, char** tokens);
void Cursor_deinit(struct Cursor* cs);
/* Cursor high level api */
char* Cursor_popToken(Args* buffs, char** stmt, char* devide);
PIKA_BOOL Cursor_isContain(char* stmt, TokenType type, char* pyload);
char* Parser_linesToAsm(Args* outBuffs, char* multiLine);
uint16_t Tokens_getSize(char* tokens) {
if (strEqu("", tokens)) {
return 0;
@ -139,7 +145,7 @@ char* strsDeleteBetween(Args* buffs_p, char* strIn, char begin, char end) {
static uint8_t Lexer_isError(char* line) {
Args buffs = {0};
uint8_t res = 0; /* not error */
char* tokens = Lexer_parseLine(&buffs, line);
char* tokens = Lexer_getTokens(&buffs, line);
if (NULL == tokens) {
res = 1; /* lex error */
goto exit;
@ -307,18 +313,38 @@ static enum StmtType Lexer_matchStmtType(char* right) {
goto exit;
}
if (is_get_string) {
/* support multi assign */
if (Cursor_isContain(right, TOKEN_devider, ",")) {
stmtType = STMT_tuple;
goto exit;
}
stmtType = STMT_string;
goto exit;
}
if (is_get_bytes) {
/* support multi assign */
if (Cursor_isContain(right, TOKEN_devider, ",")) {
stmtType = STMT_tuple;
goto exit;
}
stmtType = STMT_bytes;
goto exit;
}
if (is_get_number) {
/* support multi assign */
if (Cursor_isContain(right, TOKEN_devider, ",")) {
stmtType = STMT_tuple;
goto exit;
}
stmtType = STMT_number;
goto exit;
}
if (is_get_symbol) {
/* support multi assign */
if (Cursor_isContain(right, TOKEN_devider, ",")) {
stmtType = STMT_tuple;
goto exit;
}
stmtType = STMT_reference;
goto exit;
}
@ -337,7 +363,7 @@ char* Lexer_printTokens(Args* outBuffs, char* tokens) {
/* process */
uint16_t token_size = Tokens_getSize(tokens);
for (uint16_t i = 0; i < token_size; i++) {
char* token = Parser_popToken(&buffs, &tokens);
char* token = Tokens_pop(&buffs, &tokens);
if (token[0] == TOKEN_operator) {
printOut = strsAppend(&buffs, printOut, "{opt}");
printOut = strsAppend(&buffs, printOut, token + 1);
@ -363,10 +389,9 @@ char* Lexer_printTokens(Args* outBuffs, char* tokens) {
uint8_t Parser_checkIsDirect(char* str) {
Args buffs = {0};
char* tokens = Lexer_parseLine(&buffs, str);
uint8_t res = 0;
pika_assert(NULL != tokens);
if (Parser_isContainToken(tokens, TOKEN_operator, "=")) {
pika_assert(NULL != str);
if (Cursor_isContain(str, TOKEN_operator, "=")) {
res = 1;
goto exit;
}
@ -440,7 +465,7 @@ exit:
/* tokens is devided by space */
/* a token is [TOKENTYPE|(CONTENT)] */
char* Lexer_parseLine(Args* outBuffs, char* stmt) {
char* Lexer_getTokens(Args* outBuffs, char* stmt) {
/* init */
Arg* tokens_arg = New_arg(NULL);
tokens_arg = arg_setStr(tokens_arg, "", "");
@ -742,7 +767,7 @@ char* Lexer_parseLine(Args* outBuffs, char* stmt) {
return tokens;
}
char* Parser_popToken(Args* buffs_p, char** tokens) {
char* Tokens_pop(Args* buffs_p, char** tokens) {
return strsPopToken(buffs_p, tokens, 0x1F);
}
@ -754,7 +779,7 @@ char* Token_getPyload(char* token) {
return (char*)((uintptr_t)token + 1);
}
uint8_t Parser_isContainToken(char* tokens,
uint8_t Tokens_isContain(char* tokens,
enum TokenType token_type,
char* pyload) {
Args buffs = {0};
@ -762,7 +787,7 @@ uint8_t Parser_isContainToken(char* tokens,
uint8_t res = 0;
uint16_t token_size = Tokens_getSize(tokens);
for (int i = 0; i < token_size; i++) {
char* token = Parser_popToken(&buffs, &tokens_buff);
char* token = Tokens_pop(&buffs, &tokens_buff);
if (token_type == Token_getType(token)) {
if (strEqu(Token_getPyload(token), pyload)) {
res = 1;
@ -784,7 +809,7 @@ static const char operators[][9] = {
char* Lexer_getOperator(Args* outBuffs, char* stmt) {
Args buffs = {0};
char* operator= NULL;
char* tokens = Lexer_parseLine(&buffs, stmt);
char* tokens = Lexer_getTokens(&buffs, stmt);
// use parse state foreach to get operator
for (uint32_t i = 0; i < sizeof(operators) / 9; i++) {
@ -858,7 +883,7 @@ void Cursor_iterStart(struct Cursor* cs) {
/* token1 is the last token */
cs->token1.token = strsCopy(cs->iter_buffs, arg_getStr(cs->last_token));
/* token2 is the next token */
cs->token2.token = Parser_popToken(cs->iter_buffs, &cs->tokens);
cs->token2.token = Tokens_pop(cs->iter_buffs, &cs->tokens);
/* store last token */
arg_deinit(cs->last_token);
cs->last_token = arg_newStr(cs->token2.token);
@ -891,7 +916,7 @@ void LexToken_init(struct LexToken* lt) {
lt->type = TOKEN_strEnd;
}
void Cursor_init(struct Cursor* cs) {
void _Cursor_init(struct Cursor* cs) {
cs->tokens = NULL;
cs->length = 0;
cs->iter_index = 0;
@ -915,12 +940,12 @@ void Cursor_deinit(struct Cursor* cs) {
args_deinit(cs->buffs_p);
}
void Cursor_parse(struct Cursor* cs, char* stmt) {
void _Cursor_parse(struct Cursor* cs, char* stmt) {
if (NULL == stmt) {
cs->result = PIKA_RES_ERR_SYNTAX_ERROR;
return;
}
cs->tokens = Lexer_parseLine(cs->buffs_p, stmt);
cs->tokens = Lexer_getTokens(cs->buffs_p, stmt);
if (NULL == cs->tokens) {
cs->result = PIKA_RES_ERR_SYNTAX_ERROR;
return;
@ -928,13 +953,28 @@ void Cursor_parse(struct Cursor* cs, char* stmt) {
cs->length = Tokens_getSize(cs->tokens);
}
void Cursor_beforeIter(struct Cursor* cs) {
void _Cursor_beforeIter(struct Cursor* cs) {
/* clear first token */
if (cs->result != PIKA_RES_OK) {
return;
}
Parser_popToken(cs->buffs_p, &cs->tokens);
cs->last_token = arg_newStr(Parser_popToken(cs->buffs_p, &cs->tokens));
Tokens_pop(cs->buffs_p, &cs->tokens);
cs->last_token = arg_newStr(Tokens_pop(cs->buffs_p, &cs->tokens));
}
PIKA_BOOL Cursor_isContain(char* stmt, TokenType type, char* pyload) {
/* fast return */
if (!strstr(stmt, pyload)) {
return PIKA_FALSE;
}
Args buffs = {0};
PIKA_BOOL res = PIKA_FALSE;
char* tokens = Lexer_getTokens(&buffs, stmt);
if (Tokens_isContain(tokens, type, pyload)) {
res = PIKA_TRUE;
}
strsDeinit(&buffs);
return res;
}
char* Cursor_popToken(Args* buffs, char** tokens, char* devide) {
@ -943,12 +983,14 @@ char* Cursor_popToken(Args* buffs, char** tokens, char* devide) {
PIKA_BOOL is_find_devide = PIKA_FALSE;
Cursor_forEachToken(cs, *tokens) {
Cursor_iterStart(&cs);
if (!is_find_devide) {
if ((cs.branket_deepth == 0 && strEqu(cs.token1.pyload, devide)) ||
cs.iter_index == cs.length) {
is_find_devide = PIKA_TRUE;
Cursor_iterEnd(&cs);
continue;
}
}
if (!is_find_devide) {
out_item = arg_strAppend(out_item, cs.token1.pyload);
} else {
@ -1243,18 +1285,17 @@ uint8_t Suger_selfOperator(Args* outbuffs,
Args buffs = {0};
char _operator[2] = {0};
char* operator=(char*) _operator;
char* tokens = Lexer_parseLine(&buffs, stmt);
uint8_t is_right = 0;
if (Parser_isContainToken(tokens, TOKEN_operator, "+=")) {
if (Cursor_isContain(stmt, TOKEN_operator, "+=")) {
operator[0] = '+';
}
if (Parser_isContainToken(tokens, TOKEN_operator, "-=")) {
if (Cursor_isContain(stmt, TOKEN_operator, "-=")) {
operator[0] = '-';
}
if (Parser_isContainToken(tokens, TOKEN_operator, "*=")) {
if (Cursor_isContain(stmt, TOKEN_operator, "*=")) {
operator[0] = '*';
}
if (Parser_isContainToken(tokens, TOKEN_operator, "/=")) {
if (Cursor_isContain(stmt, TOKEN_operator, "/=")) {
operator[0] = '/';
}
/* not found self operator */
@ -1533,6 +1574,10 @@ AST* AST_parseStmt(AST* ast, char* stmt) {
}
/* match statment type */
enum StmtType stmtType = Lexer_matchStmtType(right);
if (STMT_tuple == stmtType) {
right = strsFormat(&buffs, PIKA_LINE_BUFF_SIZE, "(%s)", right);
stmtType = STMT_method;
}
/* solve operator stmt */
if (STMT_operator == stmtType) {
char* rightWithoutSubStmt = _remove_sub_stmt(&buffs, right);
@ -2018,7 +2063,7 @@ AST* AST_parseLine(char* line) {
return AST_parseLine_withBlockStack(line, NULL);
}
static char* Suger_import(Args* buffs_p, char* line) {
static char* Suger_import_as(Args* out_buffs, char* line) {
#if !PIKA_SYNTAX_IMPORT_EX_ENABLE
return line;
#endif
@ -2027,41 +2072,27 @@ static char* Suger_import(Args* buffs_p, char* line) {
char* alias = NULL;
char* origin = NULL;
char* stmt = line + 7;
/* not import, exit */
if (!strIsStartWith(line, "import ")) {
line_out = line;
goto exit;
}
Cursor_forEachToken(cs, stmt) {
Cursor_iterStart(&cs);
/* defaut set the 'origin' as the first token */
if (cs.iter_index == 1) {
origin = strsCopy(&buffs, cs.token1.pyload);
}
if (strEqu(cs.token2.pyload, " as ")) {
origin = strsCopy(&buffs, cs.token1.pyload);
}
if (strEqu(cs.token1.pyload, " as ")) {
alias = strsCopy(&buffs, cs.token2.pyload);
}
Cursor_iterEnd(&cs);
}
Cursor_deinit(&cs);
/* only import, not 'as' */
if (NULL == alias) {
if (!Cursor_isContain(stmt, TOKEN_operator, " as ")) {
line_out = line;
goto exit;
}
/* {origin} as {alias} */
origin = Cursor_popToken(&buffs, &stmt, " as ");
alias = stmt;
/* 'import' and 'as' */
line_out = strsFormat(&buffs, PIKA_LINE_BUFF_SIZE, "import %s\n%s = %s",
origin, alias, origin);
line_out = strsCopy(buffs_p, line_out);
goto exit;
exit:
strsDeinit(&buffs);
return line_out;
return strsReturnOut(&buffs, out_buffs, line_out);
}
static PIKA_BOOL _check_is_multi_assign(char* arg_list) {
@ -2144,7 +2175,7 @@ exit:
return line_out;
}
static char* Suger_from(Args* buffs_p, char* line) {
static char* Suger_from_import_as(Args* buffs_p, char* line) {
#if !PIKA_SYNTAX_IMPORT_EX_ENABLE
return line;
#endif
@ -2159,29 +2190,12 @@ static char* Suger_from(Args* buffs_p, char* line) {
goto exit;
}
Cursor_forEachToken(cs, stmt) {
Cursor_iterStart(&cs);
if (strEqu(cs.token2.pyload, " import ")) {
module = strsCopy(&buffs, cs.token1.pyload);
}
if (strEqu(cs.token1.pyload, " import ")) {
class = strsCopy(&buffs, cs.token2.pyload);
}
if (strEqu(cs.token1.pyload, " as ")) {
alias = strsCopy(&buffs, cs.token2.pyload);
}
Cursor_iterEnd(&cs);
}
Cursor_deinit(&cs);
if (NULL == module) {
line_out = strsCopy(buffs_p, "");
goto exit;
}
if (NULL == class) {
line_out = strsCopy(buffs_p, "");
goto exit;
module = Cursor_popToken(&buffs, &stmt, " import ");
if (!Cursor_isContain(stmt, TOKEN_operator, " as ")) {
class = stmt;
} else {
class = Cursor_popToken(&buffs, &stmt, " as ");
alias = stmt;
}
if (NULL == alias) {
@ -2194,27 +2208,62 @@ static char* Suger_from(Args* buffs_p, char* line) {
goto exit;
}
line_out = strsFormat(&buffs, PIKA_LINE_BUFF_SIZE, "import %s\n%s = %s.%s",
module, alias, module, class);
char* class_after = "";
while (1) {
char* class_item = Cursor_popToken(&buffs, &class, ",");
if (class_item[0] == '\0') {
break;
}
class_item = strsFormat(&buffs, PIKA_LINE_BUFF_SIZE, "%s.%s,", module,
class_item);
class_after = strsAppend(&buffs, class_after, class_item);
}
class_after[strlen(class_after) - 1] = '\0';
class = class_after;
line_out = strsFormat(&buffs, PIKA_LINE_BUFF_SIZE, "import %s\n%s = %s",
module, alias, class);
line_out = strsCopy(buffs_p, line_out);
exit:
strsDeinit(&buffs);
return line_out;
}
static char* Parser_linePreProcess(Args* buffs_p, char* line) {
static char* Suger_import(Args* buffs_p, char* line) {
line = Suger_import_as(buffs_p, line);
line = Suger_from_import_as(buffs_p, line);
return line;
}
static char* Parser_linePreProcess(Args* outbuffs, char* line) {
line = Parser_removeAnnotation(line);
Arg* line_buff = NULL;
/* check syntex error */
if (Lexer_isError(line)) {
line = NULL;
goto exit;
}
/* process EOL */
line = strsDeleteChar(buffs_p, line, '\r');
line = Suger_import(buffs_p, line);
line = Suger_from(buffs_p, line);
line = Suger_multiAssign(buffs_p, line);
line = strsDeleteChar(outbuffs, line, '\r');
/* process import */
line = Suger_import(outbuffs, line);
/* process multi assign */
int line_num = strCountSign(line, '\n') + 1;
line_buff = arg_newStr("");
for (int i = 0; i < line_num; i++) {
if (i > 0) {
line_buff = arg_strAppend(line_buff, "\n");
}
char* single_line = strsPopToken(outbuffs, &line, '\n');
single_line = Suger_multiAssign(outbuffs, single_line);
line_buff = arg_strAppend(line_buff, single_line);
}
line = strsCopy(outbuffs, arg_getStr(line_buff));
exit:
if (NULL != line_buff) {
arg_deinit(line_buff);
}
return line;
}

11
src/PikaParser.h
View File

@ -31,17 +31,18 @@
#include "dataQueueObj.h"
#include "dataStack.h"
enum TokenType {
typedef enum TokenType {
TOKEN_strEnd = 0,
TOKEN_symbol,
TOKEN_keyword,
TOKEN_operator,
TOKEN_devider,
TOKEN_literal,
};
} TokenType;
enum StmtType {
STMT_reference,
STMT_tuple,
STMT_string,
STMT_bytes,
STMT_number,
@ -107,13 +108,13 @@ ByteCodeFrame* byteCodeFrame_appendFromAsm(ByteCodeFrame* bf, char* pikaAsm);
int bytecodeFrame_fromLines(ByteCodeFrame* bytecode_frame, char* python_lines);
#define Cursor_forEach(cursor) \
Cursor_beforeIter(&cursor); \
_Cursor_beforeIter(&cursor); \
for (int __i = 0; __i < cursor.length; __i++)
#define Cursor_forEachTokenExistPs(cursor, tokens) \
/* init parserStage */ \
Cursor_init(&cursor); \
Cursor_parse(&cursor, tokens); \
_Cursor_init(&cursor); \
_Cursor_parse(&cursor, tokens); \
Cursor_forEach(cursor)
#define Cursor_forEachToken(cursor, tokens) \

2
src/PikaVM.c
View File

@ -611,7 +611,7 @@ static Arg* VM_instruction_handler_REF(PikaObj* self,
}
/* find res in host prop */
if (NULL == res){
if (NULL == res) {
res = _obj_getProp(host_object, arg_name);
}

10
src/dataString.c
View File

@ -111,7 +111,7 @@ size_t strGetSize(char* pData) {
}
char* strPointToLastToken(char* strIn, char sign) {
if(!strIsContain(strIn, sign)){
if (!strIsContain(strIn, sign)) {
return strIn;
}
int32_t size = strGetSize(strIn);
@ -191,6 +191,10 @@ int32_t strIsStartWith(char* str, char* strStart) {
/* input is null */
return 0;
}
/* fast return */
if (str[0] != strStart[0]) {
return 0;
}
uint32_t size = strGetSize(strStart);
if (0 == strncmp(str, strStart, size)) {
return 1;
@ -217,8 +221,8 @@ char* strRemovePrefix(char* inputStr, char* prefix, char* outputStr) {
}
int32_t strIsContain(char* str, char ch) {
while(*str){
if (*str == ch){
while (*str) {
if (*str == ch) {
return 1;
}
str++;

6
src/dataStrs.c
View File

@ -62,6 +62,12 @@ char* strsAppend(Args* buffs_p, char* strOrigin, char* strToAppend) {
return strOut;
}
char* strsReturnOut(Args* buffs, Args* outbuffs, char* str) {
char* line_out = strsCopy(outbuffs, str);
strsDeinit(buffs);
return line_out;
}
char* strsGetLastToken(Args* buffs_p, char* argPath, char sign) {
int32_t size = strGetSize(argPath);
char* buff = args_getBuff(buffs_p, size);

1
src/dataStrs.h
View File

@ -44,4 +44,5 @@ char* strsReplace(Args* buffs, char* orig, char* rep, char* with);
char* strsGetLine(Args* buffs, char* code);
void strsDeinit(Args* buffs);
char* strsCacheArg(Args* buffs_p, Arg* arg);
char* strsReturnOut(Args* buffs, Args* outbuffs, char* str);
#endif

1
test/compile-test.cpp
View File

@ -518,6 +518,7 @@ TEST(make, compile_depend) {
pikaMaker_setPWD(maker, "package/pikascript/");
pikaMaker_getDependencies(maker, "main");
char* uncompiled = pikaMaker_getFirstNocompiled(maker);
pika_assert(NULL != uncompiled);
pikaMaker_compileModule(maker, uncompiled);
pikaMaker_getDependencies(maker, uncompiled);
uncompiled = pikaMaker_getFirstNocompiled(maker);

226
test/parse-test.cpp
View File

@ -8,7 +8,7 @@ char* AST_genAsm(AST* ast, Args* outBuffs);
AST* AST_parseLine(char* line, Stack* blockStack);
char* Parser_LineToAsm(Args* buffs, char* line, Stack* blockStack);
int32_t AST_deinit(AST* ast);
char* Lexer_parseLine(Args* outBuffs, char* stmt);
char* Lexer_getTokens(Args* outBuffs, char* stmt);
char* Lexer_printTokens(Args* outBuffs, char* tokens);
char* strsPopTokenWithSkip_byStr(Args* buffs,
char* stmts,
@ -856,7 +856,7 @@ TEST(parser, signed_num) {
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
char* tokens_print =
Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s", tokens_print);
EXPECT_STREQ(tokens_print, "{sym}a{opt}={opt}-{lit}1\n");
printf("%s", pikaAsm);
@ -897,7 +897,7 @@ TEST(lexser, symbol_add) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, " res = add(1,2)");
char* tokens = Lexer_getTokens(buffs, " res = add(1,2)");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -916,7 +916,7 @@ TEST(lexser, symbol_1) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "a(");
char* tokens = Lexer_getTokens(buffs, "a(");
char* printTokens = Lexer_printTokens(buffs, tokens);
/* assert */
@ -933,7 +933,7 @@ TEST(lexser, operator_not) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "not not not ");
char* tokens = Lexer_getTokens(buffs, "not not not ");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -951,7 +951,7 @@ TEST(lexser, symbol_Nag) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "-10-20");
char* tokens = Lexer_getTokens(buffs, "-10-20");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -969,7 +969,7 @@ TEST(lexser, operator_all) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs,
char* tokens = Lexer_getTokens(buffs,
"not or and "
"+ += - -="
"* ** *= **="
@ -1000,7 +1000,7 @@ TEST(lexser, symbol_2) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "a+b-c(25**=ek)!=-28");
char* tokens = Lexer_getTokens(buffs, "a+b-c(25**=ek)!=-28");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -1020,7 +1020,7 @@ TEST(lexser, symbol_and) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, " res = add(1 and lkj,2)");
char* tokens = Lexer_getTokens(buffs, " res = add(1 and lkj,2)");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -1040,7 +1040,7 @@ TEST(lexser, sting) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, " a= 'elk 2'");
char* tokens = Lexer_getTokens(buffs, " a= 'elk 2'");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -1058,7 +1058,7 @@ TEST(lexser, num_1) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "1");
char* tokens = Lexer_getTokens(buffs, "1");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -1076,7 +1076,7 @@ TEST(lexser, jjcc) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "a = (1 + 1.1) * 3 - 2 /4.0");
char* tokens = Lexer_getTokens(buffs, "a = (1 + 1.1) * 3 - 2 /4.0");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -2402,7 +2402,7 @@ TEST(lexser, a_j) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "a=");
char* tokens = Lexer_getTokens(buffs, "a=");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -2479,7 +2479,7 @@ TEST(parser, a_cuohao_j) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a = (3 - 4) - 4\n";
printf("%s\n", Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines)));
printf("%s\n", Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines)));
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
EXPECT_STREQ(pikaAsm,
@ -2501,7 +2501,7 @@ TEST(parser, _3_3) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "-3+3\n";
printf("%s\n", Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines)));
printf("%s\n", Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines)));
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
EXPECT_STREQ(pikaAsm,
@ -2521,7 +2521,7 @@ TEST(parser, list_init) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a = [1, 2, 3]\n";
printf("%s\n", Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines)));
printf("%s\n", Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines)));
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
EXPECT_STREQ(pikaAsm,
@ -2541,7 +2541,7 @@ TEST(parser, list_init_fun) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "test([1, 2, 3])\n";
printf("%s\n", Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines)));
printf("%s\n", Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines)));
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
EXPECT_STREQ(pikaAsm,
@ -2562,7 +2562,7 @@ TEST(parser, bytes_iteral) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a = b'\\x00\\x01'\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2581,7 +2581,7 @@ TEST(parser, import_as) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "import PikaStdLib as std\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2602,7 +2602,7 @@ TEST(parser, str_equ) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a = 'num ='\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2621,7 +2621,7 @@ TEST(parser, bytes_index) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "res2 = b'eqrt'[2]\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2642,7 +2642,7 @@ TEST(parser, hex_iteral) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a = 0b10\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2663,7 +2663,7 @@ TEST(parser, tab) {
"for i in range(0, 100):\n"
"\tprint(i)\n"
"\n";
char* tokens_str = Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
char* tokens_str = Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\n", tokens_str);
printf("%s", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
@ -2697,7 +2697,7 @@ TEST(parser, parse_issue2) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = " recv_buf[1] = dat \n";
char* tokens = Lexer_parseLine(buffs, lines);
char* tokens = Lexer_getTokens(buffs, lines);
uint16_t token_size = Tokens_getSize(tokens);
EXPECT_EQ(token_size, 8);
char* tokens_str = Lexer_printTokens(buffs, tokens);
@ -2778,7 +2778,7 @@ TEST(parser, str_add1) {
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
char* tokens_print =
Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\r\n", tokens_print);
EXPECT_STREQ(tokens_print,
"{sym}msg{opt}={lit}\"device_names[\"{opt}+{sym}str{dvd}({sym}"
@ -2805,7 +2805,7 @@ TEST(parser, str_add2) {
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
char* tokens_print =
Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\r\n", tokens_print);
EXPECT_STREQ(tokens_print,
"{sym}msg{opt}={lit}\"device_names[\"{opt}+{sym}str{dvd}({sym}"
@ -2998,7 +2998,7 @@ TEST(parser, json_literal) {
"}'";
printf("%s\r\n", lines);
char* tokens_print =
Lexer_printTokens(buffs, Lexer_parseLine(buffs, lines));
Lexer_printTokens(buffs, Lexer_getTokens(buffs, lines));
printf("%s\r\n", tokens_print);
EXPECT_STREQ(tokens_print,
@ -3265,7 +3265,7 @@ TEST(lexser, import_issue1) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "my_import = import_test");
char* tokens = Lexer_getTokens(buffs, "my_import = import_test");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -3283,7 +3283,7 @@ TEST(lexser, dict_literal1) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(
char* tokens = Lexer_getTokens(
buffs,
"tinydict = {'name': 'runoob', 'likes': 123, 'url': 'www.runoob.com'}");
char* printTokens = Lexer_printTokens(buffs, tokens);
@ -3407,7 +3407,7 @@ TEST(lexser, function_chain) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "String('a,b,c').split(',')");
char* tokens = Lexer_getTokens(buffs, "String('a,b,c').split(',')");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -3860,7 +3860,7 @@ TEST(lexser, connet_part1) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "method(a,");
char* tokens = Lexer_getTokens(buffs, "method(a,");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -3953,7 +3953,7 @@ TEST(lexser, science_num) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "1.0e-2, 10e2, 0.1e-2, aie2, aie-2");
char* tokens = Lexer_getTokens(buffs, "1.0e-2, 10e2, 0.1e-2, aie2, aie-2");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -3973,7 +3973,7 @@ TEST(lexser, issues_I5OJQB) {
Args* buffs = New_strBuff();
/* run */
char* tokens = Lexer_parseLine(buffs, "s = '\\\\'");
char* tokens = Lexer_getTokens(buffs, "s = '\\\\'");
char* printTokens = Lexer_printTokens(buffs, tokens);
printf("%s\n", printTokens);
@ -4800,4 +4800,164 @@ TEST(parser, i_pp) {
EXPECT_EQ(pikaMemNow(), 0);
}
TEST(parser, multi_num) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "a,b ,c = 1, 2, 3\n";
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
printf("%s", pikaAsm);
EXPECT_STREQ(pikaAsm,
"B0\n"
"1 NUM 1\n"
"1 NUM 2\n"
"1 NUM 3\n"
"0 RUN \n"
"0 OUT $tmp\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 0\n"
"0 SLC \n"
"0 OUT a\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 1\n"
"0 SLC \n"
"0 OUT b\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 2\n"
"0 SLC \n"
"0 OUT c\n"
"B0\n"
"0 DEL $tmp\n"
"B0\n");
args_deinit(buffs);
EXPECT_EQ(pikaMemNow(), 0);
}
TEST(parser, multi_import) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines =
"import a,b ,c\n"
"import a,b ,c as d, e ,f\n"
"from a import b, c, d\n";
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
printf("%s", pikaAsm);
EXPECT_STREQ(pikaAsm,
"B0\n"
"0 IMP a,b,c\n"
"B0\n"
"0 IMP a,b,c\n"
"B0\n"
"1 REF a\n"
"1 REF b\n"
"1 REF c\n"
"0 RUN \n"
"0 OUT $tmp\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 0\n"
"0 SLC \n"
"0 OUT d\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 1\n"
"0 SLC \n"
"0 OUT e\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 2\n"
"0 SLC \n"
"0 OUT f\n"
"B0\n"
"0 DEL $tmp\n"
"B0\n"
"0 IMP a\n"
"B0\n"
"1 REF a.b\n"
"1 REF a.c\n"
"1 REF a.d\n"
"0 RUN \n"
"0 OUT $tmp\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 0\n"
"0 SLC \n"
"0 OUT b\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 1\n"
"0 SLC \n"
"0 OUT c\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 2\n"
"0 SLC \n"
"0 OUT d\n"
"B0\n"
"0 DEL $tmp\n"
"B0\n");
args_deinit(buffs);
EXPECT_EQ(pikaMemNow(), 0);
}
TEST(parser, from_import_as) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "from PikaStdData import String as string";
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
printf("%s", pikaAsm);
EXPECT_STREQ(pikaAsm,
"B0\n"
"0 IMP PikaStdData\n"
"B0\n"
"0 REF PikaStdData.String\n"
"0 OUT string\n"
"B0\n");
args_deinit(buffs);
EXPECT_EQ(pikaMemNow(), 0);
}
TEST(parser, multi_from_import_as) {
pikaMemInfo.heapUsedMax = 0;
Args* buffs = New_strBuff();
char* lines = "from a import b, c, d as e, f, g\n";
printf("%s\r\n", lines);
char* pikaAsm = Parser_linesToAsm(buffs, lines);
printf("%s", pikaAsm);
EXPECT_STREQ(pikaAsm,
"B0\n"
"0 IMP a\n"
"B0\n"
"1 REF a.b\n"
"1 REF a.c\n"
"1 REF a.d\n"
"0 RUN \n"
"0 OUT $tmp\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 0\n"
"0 SLC \n"
"0 OUT e\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 1\n"
"0 SLC \n"
"0 OUT f\n"
"B0\n"
"1 REF $tmp\n"
"1 NUM 2\n"
"0 SLC \n"
"0 OUT g\n"
"B0\n"
"0 DEL $tmp\n"
"B0\n");
args_deinit(buffs);
EXPECT_EQ(pikaMemNow(), 0);
}
#endif