Skip to content

Added literal list loading feature to the snapshot tool. #2422

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 16, 2018
Merged

Added literal list loading feature to the snapshot tool. #2422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions jerry-core/api/jerry-snapshot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1633,9 +1633,15 @@ jerry_parse_and_save_literals (const jerry_char_t *source_p, /**< script source
{
ecma_string_t *literal_p = ecma_get_string_from_value (*iterator_p);

/* We don't save a literal which isn't a valid identifier or it's a magic string. */
/* NOTE:
* We don't save a literal (in C format) which isn't a valid
* identifier or it's a magic string.
* TODO:
* Save all of the literals in C format as well.
*/
if (ecma_get_string_magic (literal_p) == LIT_MAGIC_STRING__COUNT
&& ecma_string_is_valid_identifier (literal_p))
&& (!is_c_format
|| (is_c_format && ecma_string_is_valid_identifier (literal_p))))
{
literal_count++;
}
Expand Down Expand Up @@ -1666,8 +1672,15 @@ jerry_parse_and_save_literals (const jerry_char_t *source_p, /**< script source
{
ecma_string_t *literal_p = ecma_get_string_from_value (*iterator_p);

/* NOTE:
* We don't save a literal (in C format) which isn't a valid
* identifier or it's a magic string.
* TODO:
* Save all of the literals in C format as well.
*/
if (ecma_get_string_magic (literal_p) == LIT_MAGIC_STRING__COUNT
&& ecma_string_is_valid_identifier (literal_p))
&& (!is_c_format
|| (is_c_format && ecma_string_is_valid_identifier (literal_p))))
{
literal_array[literal_idx++] = literal_p;
}
Expand Down
6 changes: 3 additions & 3 deletions jerry-core/lit/lit-magic-strings.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,7 @@ lit_magic_strings_ex_set (const lit_utf8_byte_t **ex_str_items, /**< character a
id < JERRY_CONTEXT (lit_magic_string_ex_count);
id = (lit_magic_string_ex_id_t) (id + 1))
{
lit_utf8_size_t string_size = lit_zt_utf8_string_size (lit_get_magic_string_ex_utf8 (id));
JERRY_ASSERT (JERRY_CONTEXT (lit_magic_string_ex_sizes)[id] == string_size);
JERRY_ASSERT (JERRY_CONTEXT (lit_magic_string_ex_sizes)[id] <= LIT_MAGIC_STRING_LENGTH_LIMIT);
lit_utf8_size_t string_size = JERRY_CONTEXT (lit_magic_string_ex_sizes)[id];
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While the change is ok, I would keep the asserts in some form. They provide useful checks (maxlimit, cesu8 format).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What should be the limit? I don't know why was it constrained to 32. Do you know the reason?
lit_zt_utf8_string_size cannot be used here, because strings can contain zeros.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Only 32? that is new to me. Btw it would be good to remove the whole macro then.
No, you can trust the size provided by the user, but you should check that the string is valid cesu8.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added the check. Please check.


/**
* Check whether the strings are sorted by size and lexicographically,
Expand All @@ -163,6 +161,8 @@ lit_magic_strings_ex_set (const lit_utf8_byte_t **ex_str_items, /**< character a
{
const lit_magic_string_ex_id_t prev_id = id - 1;
const lit_utf8_size_t prev_string_size = lit_get_magic_string_ex_size (prev_id);
JERRY_ASSERT (lit_is_valid_cesu8_string (lit_get_magic_string_ex_utf8 (id),
string_size));
JERRY_ASSERT (prev_string_size <= string_size);

if (prev_string_size == string_size)
Expand Down
5 changes: 0 additions & 5 deletions jerry-core/lit/lit-magic-strings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,6 @@

#include "lit-globals.h"

/**
* Limit for magic string length
*/
#define LIT_MAGIC_STRING_LENGTH_LIMIT 32

/**
* Identifiers of ECMA and implementation-defined magic string constants
*/
Expand Down
70 changes: 58 additions & 12 deletions jerry-main/main-unix-snapshot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/

#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include "jerryscript.h"
Expand All @@ -27,15 +28,22 @@
*/
#define JERRY_BUFFER_SIZE (1048576)

/**
* Maximum number of loaded literals
*/
#define JERRY_LITERAL_LENGTH (4096)

/**
* Standalone Jerry exit codes
*/
#define JERRY_STANDALONE_EXIT_CODE_OK (0)
#define JERRY_STANDALONE_EXIT_CODE_FAIL (1)

static uint8_t input_buffer[ JERRY_BUFFER_SIZE ];
static uint32_t output_buffer[ JERRY_BUFFER_SIZE / 4 ];
static uint8_t input_buffer[JERRY_BUFFER_SIZE];
static uint32_t output_buffer[JERRY_BUFFER_SIZE / 4];
static const char *output_file_name_p = "js.snapshot";
static jerry_length_t magic_string_lengths[JERRY_LITERAL_LENGTH];
static jerry_char_ptr_t magic_string_items[JERRY_LITERAL_LENGTH];

/**
* Check whether JerryScript has a requested feature enabled or not. If not,
Expand Down Expand Up @@ -168,6 +176,7 @@ typedef enum
OPT_GENERATE_LITERAL_C,
OPT_GENERATE_SHOW_OP,
OPT_GENERATE_OUT,
OPT_IMPORT_LITERAL_LIST
} generate_opt_id_t;

/**
Expand All @@ -179,6 +188,9 @@ static const cli_opt_t generate_opts[] =
.help = "print this help and exit"),
CLI_OPT_DEF (.id = OPT_GENERATE_STATIC, .opt = "s", .longopt = "static",
.help = "generate static snapshot"),
CLI_OPT_DEF (.id = OPT_IMPORT_LITERAL_LIST, .longopt = "load-literals-list-format",
.meta = "FILE",
.help = "import literals from list format (for static snapshots)"),
CLI_OPT_DEF (.id = OPT_GENERATE_LITERAL_LIST, .longopt = "save-literals-list-format",
.meta = "FILE",
.help = "export literals found in parsed JS input (in list format)"),
Expand All @@ -190,7 +202,7 @@ static const cli_opt_t generate_opts[] =
CLI_OPT_DEF (.id = OPT_GENERATE_OUT, .opt = "o", .meta="FILE",
.help = "specify output file name (default: js.snapshot)"),
CLI_OPT_DEF (.id = CLI_OPT_DEFAULT, .meta = "FILE",
.help = "input snapshot file")
.help = "input source file")
};

/**
Expand All @@ -206,13 +218,14 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
(void) argc;

bool is_save_literals_mode_in_c_format = false;
bool is_import_literals = false;
uint32_t snapshot_flags = 0;
jerry_init_flag_t flags = JERRY_INIT_EMPTY;

const char *file_name_p = NULL;
uint8_t *source_p = input_buffer;
size_t source_length = 0;
const char *save_literals_file_name_p = NULL;
const char *literals_file_name_p = NULL;

cli_change_opts (cli_state_p, generate_opts);

Expand All @@ -230,17 +243,19 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
snapshot_flags |= JERRY_SNAPSHOT_SAVE_STATIC;
break;
}
case OPT_GENERATE_LITERAL_LIST:
case OPT_GENERATE_LITERAL_C:
case OPT_GENERATE_LITERAL_LIST:
case OPT_IMPORT_LITERAL_LIST:
{
if (save_literals_file_name_p != NULL)
if (literals_file_name_p != NULL)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cannot we both load and save literals?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is no make sense

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why? The loaded and saved list can be different. Although loading probably used only with static snapshots.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can change this, but it is not a valid use case. It will never happen to use them in the same command.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok keep it this way for now.

{
jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: literal file name already specified");
return JERRY_STANDALONE_EXIT_CODE_FAIL;
}

is_import_literals = (id == OPT_IMPORT_LITERAL_LIST);
is_save_literals_mode_in_c_format = (id == OPT_GENERATE_LITERAL_C);
save_literals_file_name_p = cli_consume_string (cli_state_p);
literals_file_name_p = cli_consume_string (cli_state_p);
break;
}
case OPT_GENERATE_SHOW_OP:
Expand Down Expand Up @@ -306,8 +321,39 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
return JERRY_STANDALONE_EXIT_CODE_FAIL;
}

jerry_value_t snapshot_result;
if (is_import_literals)
{
uint8_t *sp_buffer_start_p = source_p + source_length + 1;
size_t sp_buffer_size = read_file (sp_buffer_start_p, literals_file_name_p);

if (sp_buffer_size > 0)
{
const char *sp_buffer_p = (const char *) sp_buffer_start_p;
uint32_t num_of_lit = 0;

do
{
char *sp_buffer_end_p = NULL;
jerry_length_t mstr_size = (jerry_length_t) strtol (sp_buffer_p, &sp_buffer_end_p, 10);
if (mstr_size > 0)
{
magic_string_items[num_of_lit] = (jerry_char_ptr_t) (sp_buffer_end_p + 1);
magic_string_lengths[num_of_lit] = mstr_size;
num_of_lit++;
}
sp_buffer_p = sp_buffer_end_p + mstr_size + 1;
}
while ((size_t) (sp_buffer_p - (char *) sp_buffer_start_p) < sp_buffer_size);

if (num_of_lit > 0)
{
jerry_register_magic_strings (magic_string_items, num_of_lit,
magic_string_lengths);
}
}
}

jerry_value_t snapshot_result;
snapshot_result = jerry_generate_snapshot ((jerry_char_t *) file_name_p,
(size_t) strlen (file_name_p),
(jerry_char_t *) source_p,
Expand Down Expand Up @@ -343,7 +389,7 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */

printf ("Created snapshot file: '%s' (%lu bytes)\n", output_file_name_p, (unsigned long) snapshot_size);

if (save_literals_file_name_p != NULL)
if (literals_file_name_p != NULL && !is_import_literals)
{
const size_t literal_buffer_size = jerry_parse_and_save_literals ((jerry_char_t *) source_p,
source_length,
Expand All @@ -357,18 +403,18 @@ process_generate (cli_state_t *cli_state_p, /**< cli state */
return JERRY_STANDALONE_EXIT_CODE_FAIL;
}

FILE *literal_file_p = fopen (save_literals_file_name_p, "wb");
FILE *literal_file_p = fopen (literals_file_name_p, "wb");

if (literal_file_p == NULL)
{
jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: Unable to write literal file: '%s'\n", save_literals_file_name_p);
jerry_port_log (JERRY_LOG_LEVEL_ERROR, "Error: Unable to write literal file: '%s'\n", literals_file_name_p);
return JERRY_STANDALONE_EXIT_CODE_FAIL;
}

fwrite (output_buffer, sizeof (uint8_t), literal_buffer_size, literal_file_p);
fclose (literal_file_p);

printf ("Created literal file: '%s' (%lu bytes)\n", save_literals_file_name_p, (unsigned long) literal_buffer_size);
printf ("Created literal file: '%s' (%lu bytes)\n", literals_file_name_p, (unsigned long) literal_buffer_size);
}

return 0;
Expand Down