diff options
Diffstat (limited to 'scripts/kallsyms.c')
| -rw-r--r-- | scripts/kallsyms.c | 427 |
1 files changed, 98 insertions, 329 deletions
diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 9be41a9f5aff..d591578bd3b2 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c | |||
| @@ -24,75 +24,37 @@ | |||
| 24 | * | 24 | * |
| 25 | */ | 25 | */ |
| 26 | 26 | ||
| 27 | #define _GNU_SOURCE | ||
| 28 | |||
| 27 | #include <stdio.h> | 29 | #include <stdio.h> |
| 28 | #include <stdlib.h> | 30 | #include <stdlib.h> |
| 29 | #include <string.h> | 31 | #include <string.h> |
| 30 | #include <ctype.h> | 32 | #include <ctype.h> |
| 31 | 33 | ||
| 32 | /* maximum token length used. It doesn't pay to increase it a lot, because | ||
| 33 | * very long substrings probably don't repeat themselves too often. */ | ||
| 34 | #define MAX_TOK_SIZE 11 | ||
| 35 | #define KSYM_NAME_LEN 127 | 34 | #define KSYM_NAME_LEN 127 |
| 36 | 35 | ||
| 37 | /* we use only a subset of the complete symbol table to gather the token count, | ||
| 38 | * to speed up compression, at the expense of a little compression ratio */ | ||
| 39 | #define WORKING_SET 1024 | ||
| 40 | |||
| 41 | /* first find the best token only on the list of tokens that would profit more | ||
| 42 | * than GOOD_BAD_THRESHOLD. Only if this list is empty go to the "bad" list. | ||
| 43 | * Increasing this value will put less tokens on the "good" list, so the search | ||
| 44 | * is faster. However, if the good list runs out of tokens, we must painfully | ||
| 45 | * search the bad list. */ | ||
| 46 | #define GOOD_BAD_THRESHOLD 10 | ||
| 47 | |||
| 48 | /* token hash parameters */ | ||
| 49 | #define HASH_BITS 18 | ||
| 50 | #define HASH_TABLE_SIZE (1 << HASH_BITS) | ||
| 51 | #define HASH_MASK (HASH_TABLE_SIZE - 1) | ||
| 52 | #define HASH_BASE_OFFSET 2166136261U | ||
| 53 | #define HASH_FOLD(a) ((a)&(HASH_MASK)) | ||
| 54 | |||
| 55 | /* flags to mark symbols */ | ||
| 56 | #define SYM_FLAG_VALID 1 | ||
| 57 | #define SYM_FLAG_SAMPLED 2 | ||
| 58 | 36 | ||
| 59 | struct sym_entry { | 37 | struct sym_entry { |
| 60 | unsigned long long addr; | 38 | unsigned long long addr; |
| 61 | char type; | 39 | unsigned int len; |
| 62 | unsigned char flags; | ||
| 63 | unsigned char len; | ||
| 64 | unsigned char *sym; | 40 | unsigned char *sym; |
| 65 | }; | 41 | }; |
| 66 | 42 | ||
| 67 | 43 | ||
| 68 | static struct sym_entry *table; | 44 | static struct sym_entry *table; |
| 69 | static int size, cnt; | 45 | static unsigned int table_size, table_cnt; |
| 70 | static unsigned long long _stext, _etext, _sinittext, _einittext, _sextratext, _eextratext; | 46 | static unsigned long long _stext, _etext, _sinittext, _einittext, _sextratext, _eextratext; |
| 71 | static int all_symbols = 0; | 47 | static int all_symbols = 0; |
| 72 | static char symbol_prefix_char = '\0'; | 48 | static char symbol_prefix_char = '\0'; |
| 73 | 49 | ||
| 74 | struct token { | 50 | int token_profit[0x10000]; |
| 75 | unsigned char data[MAX_TOK_SIZE]; | ||
| 76 | unsigned char len; | ||
| 77 | /* profit: the number of bytes that could be saved by inserting this | ||
| 78 | * token into the table */ | ||
| 79 | int profit; | ||
| 80 | struct token *next; /* next token on the hash list */ | ||
| 81 | struct token *right; /* next token on the good/bad list */ | ||
| 82 | struct token *left; /* previous token on the good/bad list */ | ||
| 83 | struct token *smaller; /* token that is less one letter than this one */ | ||
| 84 | }; | ||
| 85 | |||
| 86 | struct token bad_head, good_head; | ||
| 87 | struct token *hash_table[HASH_TABLE_SIZE]; | ||
| 88 | 51 | ||
| 89 | /* the table that holds the result of the compression */ | 52 | /* the table that holds the result of the compression */ |
| 90 | unsigned char best_table[256][MAX_TOK_SIZE+1]; | 53 | unsigned char best_table[256][2]; |
| 91 | unsigned char best_table_len[256]; | 54 | unsigned char best_table_len[256]; |
| 92 | 55 | ||
| 93 | 56 | ||
| 94 | static void | 57 | static void usage(void) |
| 95 | usage(void) | ||
| 96 | { | 58 | { |
| 97 | fprintf(stderr, "Usage: kallsyms [--all-symbols] [--symbol-prefix=<prefix char>] < in.map > out.S\n"); | 59 | fprintf(stderr, "Usage: kallsyms [--all-symbols] [--symbol-prefix=<prefix char>] < in.map > out.S\n"); |
| 98 | exit(1); | 60 | exit(1); |
| @@ -102,21 +64,19 @@ usage(void) | |||
| 102 | * This ignores the intensely annoying "mapping symbols" found | 64 | * This ignores the intensely annoying "mapping symbols" found |
| 103 | * in ARM ELF files: $a, $t and $d. | 65 | * in ARM ELF files: $a, $t and $d. |
| 104 | */ | 66 | */ |
| 105 | static inline int | 67 | static inline int is_arm_mapping_symbol(const char *str) |
| 106 | is_arm_mapping_symbol(const char *str) | ||
| 107 | { | 68 | { |
| 108 | return str[0] == '$' && strchr("atd", str[1]) | 69 | return str[0] == '$' && strchr("atd", str[1]) |
| 109 | && (str[2] == '\0' || str[2] == '.'); | 70 | && (str[2] == '\0' || str[2] == '.'); |
| 110 | } | 71 | } |
| 111 | 72 | ||
| 112 | static int | 73 | static int read_symbol(FILE *in, struct sym_entry *s) |
| 113 | read_symbol(FILE *in, struct sym_entry *s) | ||
| 114 | { | 74 | { |
| 115 | char str[500]; | 75 | char str[500]; |
| 116 | char *sym; | 76 | char *sym, stype; |
| 117 | int rc; | 77 | int rc; |
| 118 | 78 | ||
| 119 | rc = fscanf(in, "%llx %c %499s\n", &s->addr, &s->type, str); | 79 | rc = fscanf(in, "%llx %c %499s\n", &s->addr, &stype, str); |
| 120 | if (rc != 3) { | 80 | if (rc != 3) { |
| 121 | if (rc != EOF) { | 81 | if (rc != EOF) { |
| 122 | /* skip line */ | 82 | /* skip line */ |
| @@ -143,7 +103,7 @@ read_symbol(FILE *in, struct sym_entry *s) | |||
| 143 | _sextratext = s->addr; | 103 | _sextratext = s->addr; |
| 144 | else if (strcmp(sym, "_eextratext") == 0) | 104 | else if (strcmp(sym, "_eextratext") == 0) |
| 145 | _eextratext = s->addr; | 105 | _eextratext = s->addr; |
| 146 | else if (toupper(s->type) == 'A') | 106 | else if (toupper(stype) == 'A') |
| 147 | { | 107 | { |
| 148 | /* Keep these useful absolute symbols */ | 108 | /* Keep these useful absolute symbols */ |
| 149 | if (strcmp(sym, "__kernel_syscall_via_break") && | 109 | if (strcmp(sym, "__kernel_syscall_via_break") && |
| @@ -153,22 +113,24 @@ read_symbol(FILE *in, struct sym_entry *s) | |||
| 153 | return -1; | 113 | return -1; |
| 154 | 114 | ||
| 155 | } | 115 | } |
| 156 | else if (toupper(s->type) == 'U' || | 116 | else if (toupper(stype) == 'U' || |
| 157 | is_arm_mapping_symbol(sym)) | 117 | is_arm_mapping_symbol(sym)) |
| 158 | return -1; | 118 | return -1; |
| 119 | /* exclude also MIPS ELF local symbols ($L123 instead of .L123) */ | ||
| 120 | else if (str[0] == '$') | ||
| 121 | return -1; | ||
| 159 | 122 | ||
| 160 | /* include the type field in the symbol name, so that it gets | 123 | /* include the type field in the symbol name, so that it gets |
| 161 | * compressed together */ | 124 | * compressed together */ |
| 162 | s->len = strlen(str) + 1; | 125 | s->len = strlen(str) + 1; |
| 163 | s->sym = (char *) malloc(s->len + 1); | 126 | s->sym = malloc(s->len + 1); |
| 164 | strcpy(s->sym + 1, str); | 127 | strcpy((char *)s->sym + 1, str); |
| 165 | s->sym[0] = s->type; | 128 | s->sym[0] = stype; |
| 166 | 129 | ||
| 167 | return 0; | 130 | return 0; |
| 168 | } | 131 | } |
| 169 | 132 | ||
| 170 | static int | 133 | static int symbol_valid(struct sym_entry *s) |
| 171 | symbol_valid(struct sym_entry *s) | ||
| 172 | { | 134 | { |
| 173 | /* Symbols which vary between passes. Passes 1 and 2 must have | 135 | /* Symbols which vary between passes. Passes 1 and 2 must have |
| 174 | * identical symbol lists. The kallsyms_* symbols below are only added | 136 | * identical symbol lists. The kallsyms_* symbols below are only added |
| @@ -214,30 +176,29 @@ symbol_valid(struct sym_entry *s) | |||
| 214 | } | 176 | } |
| 215 | 177 | ||
| 216 | /* Exclude symbols which vary between passes. */ | 178 | /* Exclude symbols which vary between passes. */ |
| 217 | if (strstr(s->sym + offset, "_compiled.")) | 179 | if (strstr((char *)s->sym + offset, "_compiled.")) |
| 218 | return 0; | 180 | return 0; |
| 219 | 181 | ||
| 220 | for (i = 0; special_symbols[i]; i++) | 182 | for (i = 0; special_symbols[i]; i++) |
| 221 | if( strcmp(s->sym + offset, special_symbols[i]) == 0 ) | 183 | if( strcmp((char *)s->sym + offset, special_symbols[i]) == 0 ) |
| 222 | return 0; | 184 | return 0; |
| 223 | 185 | ||
| 224 | return 1; | 186 | return 1; |
| 225 | } | 187 | } |
| 226 | 188 | ||
| 227 | static void | 189 | static void read_map(FILE *in) |
| 228 | read_map(FILE *in) | ||
| 229 | { | 190 | { |
| 230 | while (!feof(in)) { | 191 | while (!feof(in)) { |
| 231 | if (cnt >= size) { | 192 | if (table_cnt >= table_size) { |
| 232 | size += 10000; | 193 | table_size += 10000; |
| 233 | table = realloc(table, sizeof(*table) * size); | 194 | table = realloc(table, sizeof(*table) * table_size); |
| 234 | if (!table) { | 195 | if (!table) { |
| 235 | fprintf(stderr, "out of memory\n"); | 196 | fprintf(stderr, "out of memory\n"); |
| 236 | exit (1); | 197 | exit (1); |
| 237 | } | 198 | } |
| 238 | } | 199 | } |
| 239 | if (read_symbol(in, &table[cnt]) == 0) | 200 | if (read_symbol(in, &table[table_cnt]) == 0) |
| 240 | cnt++; | 201 | table_cnt++; |
| 241 | } | 202 | } |
| 242 | } | 203 | } |
| 243 | 204 | ||
| @@ -281,10 +242,9 @@ static int expand_symbol(unsigned char *data, int len, char *result) | |||
| 281 | return total; | 242 | return total; |
| 282 | } | 243 | } |
| 283 | 244 | ||
| 284 | static void | 245 | static void write_src(void) |
| 285 | write_src(void) | ||
| 286 | { | 246 | { |
| 287 | int i, k, off, valid; | 247 | unsigned int i, k, off; |
| 288 | unsigned int best_idx[256]; | 248 | unsigned int best_idx[256]; |
| 289 | unsigned int *markers; | 249 | unsigned int *markers; |
| 290 | char buf[KSYM_NAME_LEN+1]; | 250 | char buf[KSYM_NAME_LEN+1]; |
| @@ -301,33 +261,24 @@ write_src(void) | |||
| 301 | printf(".data\n"); | 261 | printf(".data\n"); |
| 302 | 262 | ||
| 303 | output_label("kallsyms_addresses"); | 263 | output_label("kallsyms_addresses"); |
| 304 | valid = 0; | 264 | for (i = 0; i < table_cnt; i++) { |
| 305 | for (i = 0; i < cnt; i++) { | 265 | printf("\tPTR\t%#llx\n", table[i].addr); |
| 306 | if (table[i].flags & SYM_FLAG_VALID) { | ||
| 307 | printf("\tPTR\t%#llx\n", table[i].addr); | ||
| 308 | valid++; | ||
| 309 | } | ||
| 310 | } | 266 | } |
| 311 | printf("\n"); | 267 | printf("\n"); |
| 312 | 268 | ||
| 313 | output_label("kallsyms_num_syms"); | 269 | output_label("kallsyms_num_syms"); |
| 314 | printf("\tPTR\t%d\n", valid); | 270 | printf("\tPTR\t%d\n", table_cnt); |
| 315 | printf("\n"); | 271 | printf("\n"); |
| 316 | 272 | ||
| 317 | /* table of offset markers, that give the offset in the compressed stream | 273 | /* table of offset markers, that give the offset in the compressed stream |
| 318 | * every 256 symbols */ | 274 | * every 256 symbols */ |
| 319 | markers = (unsigned int *) malloc(sizeof(unsigned int)*((valid + 255) / 256)); | 275 | markers = (unsigned int *) malloc(sizeof(unsigned int) * ((table_cnt + 255) / 256)); |
| 320 | 276 | ||
| 321 | output_label("kallsyms_names"); | 277 | output_label("kallsyms_names"); |
| 322 | valid = 0; | ||
| 323 | off = 0; | 278 | off = 0; |
| 324 | for (i = 0; i < cnt; i++) { | 279 | for (i = 0; i < table_cnt; i++) { |
| 325 | 280 | if ((i & 0xFF) == 0) | |
| 326 | if (!table[i].flags & SYM_FLAG_VALID) | 281 | markers[i >> 8] = off; |
| 327 | continue; | ||
| 328 | |||
| 329 | if ((valid & 0xFF) == 0) | ||
| 330 | markers[valid >> 8] = off; | ||
| 331 | 282 | ||
| 332 | printf("\t.byte 0x%02x", table[i].len); | 283 | printf("\t.byte 0x%02x", table[i].len); |
| 333 | for (k = 0; k < table[i].len; k++) | 284 | for (k = 0; k < table[i].len; k++) |
| @@ -335,12 +286,11 @@ write_src(void) | |||
| 335 | printf("\n"); | 286 | printf("\n"); |
| 336 | 287 | ||
| 337 | off += table[i].len + 1; | 288 | off += table[i].len + 1; |
| 338 | valid++; | ||
| 339 | } | 289 | } |
| 340 | printf("\n"); | 290 | printf("\n"); |
| 341 | 291 | ||
| 342 | output_label("kallsyms_markers"); | 292 | output_label("kallsyms_markers"); |
| 343 | for (i = 0; i < ((valid + 255) >> 8); i++) | 293 | for (i = 0; i < ((table_cnt + 255) >> 8); i++) |
| 344 | printf("\tPTR\t%d\n", markers[i]); | 294 | printf("\tPTR\t%d\n", markers[i]); |
| 345 | printf("\n"); | 295 | printf("\n"); |
| 346 | 296 | ||
| @@ -350,7 +300,7 @@ write_src(void) | |||
| 350 | off = 0; | 300 | off = 0; |
| 351 | for (i = 0; i < 256; i++) { | 301 | for (i = 0; i < 256; i++) { |
| 352 | best_idx[i] = off; | 302 | best_idx[i] = off; |
| 353 | expand_symbol(best_table[i],best_table_len[i],buf); | 303 | expand_symbol(best_table[i], best_table_len[i], buf); |
| 354 | printf("\t.asciz\t\"%s\"\n", buf); | 304 | printf("\t.asciz\t\"%s\"\n", buf); |
| 355 | off += strlen(buf) + 1; | 305 | off += strlen(buf) + 1; |
| 356 | } | 306 | } |
| @@ -365,153 +315,13 @@ write_src(void) | |||
| 365 | 315 | ||
| 366 | /* table lookup compression functions */ | 316 | /* table lookup compression functions */ |
| 367 | 317 | ||
| 368 | static inline unsigned int rehash_token(unsigned int hash, unsigned char data) | ||
| 369 | { | ||
| 370 | return ((hash * 16777619) ^ data); | ||
| 371 | } | ||
| 372 | |||
| 373 | static unsigned int hash_token(unsigned char *data, int len) | ||
| 374 | { | ||
| 375 | unsigned int hash=HASH_BASE_OFFSET; | ||
| 376 | int i; | ||
| 377 | |||
| 378 | for (i = 0; i < len; i++) | ||
| 379 | hash = rehash_token(hash, data[i]); | ||
| 380 | |||
| 381 | return HASH_FOLD(hash); | ||
| 382 | } | ||
| 383 | |||
| 384 | /* find a token given its data and hash value */ | ||
| 385 | static struct token *find_token_hash(unsigned char *data, int len, unsigned int hash) | ||
| 386 | { | ||
| 387 | struct token *ptr; | ||
| 388 | |||
| 389 | ptr = hash_table[hash]; | ||
| 390 | |||
| 391 | while (ptr) { | ||
| 392 | if ((ptr->len == len) && (memcmp(ptr->data, data, len) == 0)) | ||
| 393 | return ptr; | ||
| 394 | ptr=ptr->next; | ||
| 395 | } | ||
| 396 | |||
| 397 | return NULL; | ||
| 398 | } | ||
| 399 | |||
| 400 | static inline void insert_token_in_group(struct token *head, struct token *ptr) | ||
| 401 | { | ||
| 402 | ptr->right = head->right; | ||
| 403 | ptr->right->left = ptr; | ||
| 404 | head->right = ptr; | ||
| 405 | ptr->left = head; | ||
| 406 | } | ||
| 407 | |||
| 408 | static inline void remove_token_from_group(struct token *ptr) | ||
| 409 | { | ||
| 410 | ptr->left->right = ptr->right; | ||
| 411 | ptr->right->left = ptr->left; | ||
| 412 | } | ||
| 413 | |||
| 414 | |||
| 415 | /* build the counts for all the tokens that start with "data", and have lenghts | ||
| 416 | * from 2 to "len" */ | ||
| 417 | static void learn_token(unsigned char *data, int len) | ||
| 418 | { | ||
| 419 | struct token *ptr,*last_ptr; | ||
| 420 | int i, newprofit; | ||
| 421 | unsigned int hash = HASH_BASE_OFFSET; | ||
| 422 | unsigned int hashes[MAX_TOK_SIZE + 1]; | ||
| 423 | |||
| 424 | if (len > MAX_TOK_SIZE) | ||
| 425 | len = MAX_TOK_SIZE; | ||
| 426 | |||
| 427 | /* calculate and store the hash values for all the sub-tokens */ | ||
| 428 | hash = rehash_token(hash, data[0]); | ||
| 429 | for (i = 2; i <= len; i++) { | ||
| 430 | hash = rehash_token(hash, data[i-1]); | ||
| 431 | hashes[i] = HASH_FOLD(hash); | ||
| 432 | } | ||
| 433 | |||
| 434 | last_ptr = NULL; | ||
| 435 | ptr = NULL; | ||
| 436 | |||
| 437 | for (i = len; i >= 2; i--) { | ||
| 438 | hash = hashes[i]; | ||
| 439 | |||
| 440 | if (!ptr) ptr = find_token_hash(data, i, hash); | ||
| 441 | |||
| 442 | if (!ptr) { | ||
| 443 | /* create a new token entry */ | ||
| 444 | ptr = (struct token *) malloc(sizeof(*ptr)); | ||
| 445 | |||
| 446 | memcpy(ptr->data, data, i); | ||
| 447 | ptr->len = i; | ||
| 448 | |||
| 449 | /* when we create an entry, it's profit is 0 because | ||
| 450 | * we also take into account the size of the token on | ||
| 451 | * the compressed table. We then subtract GOOD_BAD_THRESHOLD | ||
| 452 | * so that the test to see if this token belongs to | ||
| 453 | * the good or bad list, is a comparison to zero */ | ||
| 454 | ptr->profit = -GOOD_BAD_THRESHOLD; | ||
| 455 | |||
| 456 | ptr->next = hash_table[hash]; | ||
| 457 | hash_table[hash] = ptr; | ||
| 458 | |||
| 459 | insert_token_in_group(&bad_head, ptr); | ||
| 460 | |||
| 461 | ptr->smaller = NULL; | ||
| 462 | } else { | ||
| 463 | newprofit = ptr->profit + (ptr->len - 1); | ||
| 464 | /* check to see if this token needs to be moved to a | ||
| 465 | * different list */ | ||
| 466 | if((ptr->profit < 0) && (newprofit >= 0)) { | ||
| 467 | remove_token_from_group(ptr); | ||
| 468 | insert_token_in_group(&good_head,ptr); | ||
| 469 | } | ||
| 470 | ptr->profit = newprofit; | ||
| 471 | } | ||
| 472 | |||
| 473 | if (last_ptr) last_ptr->smaller = ptr; | ||
| 474 | last_ptr = ptr; | ||
| 475 | |||
| 476 | ptr = ptr->smaller; | ||
| 477 | } | ||
| 478 | } | ||
| 479 | |||
| 480 | /* decrease the counts for all the tokens that start with "data", and have lenghts | ||
| 481 | * from 2 to "len". This function is much simpler than learn_token because we have | ||
| 482 | * more guarantees (tho tokens exist, the ->smaller pointer is set, etc.) | ||
| 483 | * The two separate functions exist only because of compression performance */ | ||
| 484 | static void forget_token(unsigned char *data, int len) | ||
| 485 | { | ||
| 486 | struct token *ptr; | ||
| 487 | int i, newprofit; | ||
| 488 | unsigned int hash=0; | ||
| 489 | |||
| 490 | if (len > MAX_TOK_SIZE) len = MAX_TOK_SIZE; | ||
| 491 | |||
| 492 | hash = hash_token(data, len); | ||
| 493 | ptr = find_token_hash(data, len, hash); | ||
| 494 | |||
| 495 | for (i = len; i >= 2; i--) { | ||
| 496 | |||
| 497 | newprofit = ptr->profit - (ptr->len - 1); | ||
| 498 | if ((ptr->profit >= 0) && (newprofit < 0)) { | ||
| 499 | remove_token_from_group(ptr); | ||
| 500 | insert_token_in_group(&bad_head, ptr); | ||
| 501 | } | ||
| 502 | ptr->profit=newprofit; | ||
| 503 | |||
| 504 | ptr=ptr->smaller; | ||
| 505 | } | ||
| 506 | } | ||
| 507 | |||
| 508 | /* count all the possible tokens in a symbol */ | 318 | /* count all the possible tokens in a symbol */ |
| 509 | static void learn_symbol(unsigned char *symbol, int len) | 319 | static void learn_symbol(unsigned char *symbol, int len) |
| 510 | { | 320 | { |
| 511 | int i; | 321 | int i; |
| 512 | 322 | ||
| 513 | for (i = 0; i < len - 1; i++) | 323 | for (i = 0; i < len - 1; i++) |
| 514 | learn_token(symbol + i, len - i); | 324 | token_profit[ symbol[i] + (symbol[i + 1] << 8) ]++; |
| 515 | } | 325 | } |
| 516 | 326 | ||
| 517 | /* decrease the count for all the possible tokens in a symbol */ | 327 | /* decrease the count for all the possible tokens in a symbol */ |
| @@ -520,117 +330,90 @@ static void forget_symbol(unsigned char *symbol, int len) | |||
| 520 | int i; | 330 | int i; |
| 521 | 331 | ||
| 522 | for (i = 0; i < len - 1; i++) | 332 | for (i = 0; i < len - 1; i++) |
| 523 | forget_token(symbol + i, len - i); | 333 | token_profit[ symbol[i] + (symbol[i + 1] << 8) ]--; |
| 524 | } | 334 | } |
| 525 | 335 | ||
| 526 | /* set all the symbol flags and do the initial token count */ | 336 | /* remove all the invalid symbols from the table and do the initial token count */ |
| 527 | static void build_initial_tok_table(void) | 337 | static void build_initial_tok_table(void) |
| 528 | { | 338 | { |
| 529 | int i, use_it, valid; | 339 | unsigned int i, pos; |
| 530 | 340 | ||
| 531 | valid = 0; | 341 | pos = 0; |
| 532 | for (i = 0; i < cnt; i++) { | 342 | for (i = 0; i < table_cnt; i++) { |
| 533 | table[i].flags = 0; | ||
| 534 | if ( symbol_valid(&table[i]) ) { | 343 | if ( symbol_valid(&table[i]) ) { |
| 535 | table[i].flags |= SYM_FLAG_VALID; | 344 | if (pos != i) |
| 536 | valid++; | 345 | table[pos] = table[i]; |
| 346 | learn_symbol(table[pos].sym, table[pos].len); | ||
| 347 | pos++; | ||
| 537 | } | 348 | } |
| 538 | } | 349 | } |
| 539 | 350 | table_cnt = pos; | |
| 540 | use_it = 0; | ||
| 541 | for (i = 0; i < cnt; i++) { | ||
| 542 | |||
| 543 | /* subsample the available symbols. This method is almost like | ||
| 544 | * a Bresenham's algorithm to get uniformly distributed samples | ||
| 545 | * across the symbol table */ | ||
| 546 | if (table[i].flags & SYM_FLAG_VALID) { | ||
| 547 | |||
| 548 | use_it += WORKING_SET; | ||
| 549 | |||
| 550 | if (use_it >= valid) { | ||
| 551 | table[i].flags |= SYM_FLAG_SAMPLED; | ||
| 552 | use_it -= valid; | ||
| 553 | } | ||
| 554 | } | ||
| 555 | if (table[i].flags & SYM_FLAG_SAMPLED) | ||
| 556 | learn_symbol(table[i].sym, table[i].len); | ||
| 557 | } | ||
| 558 | } | 351 | } |
| 559 | 352 | ||
| 560 | /* replace a given token in all the valid symbols. Use the sampled symbols | 353 | /* replace a given token in all the valid symbols. Use the sampled symbols |
| 561 | * to update the counts */ | 354 | * to update the counts */ |
| 562 | static void compress_symbols(unsigned char *str, int tlen, int idx) | 355 | static void compress_symbols(unsigned char *str, int idx) |
| 563 | { | 356 | { |
| 564 | int i, len, learn, size; | 357 | unsigned int i, len, size; |
| 565 | unsigned char *p; | 358 | unsigned char *p1, *p2; |
| 566 | 359 | ||
| 567 | for (i = 0; i < cnt; i++) { | 360 | for (i = 0; i < table_cnt; i++) { |
| 568 | |||
| 569 | if (!(table[i].flags & SYM_FLAG_VALID)) continue; | ||
| 570 | 361 | ||
| 571 | len = table[i].len; | 362 | len = table[i].len; |
| 572 | learn = 0; | 363 | p1 = table[i].sym; |
| 573 | p = table[i].sym; | 364 | |
| 365 | /* find the token on the symbol */ | ||
| 366 | p2 = memmem(p1, len, str, 2); | ||
| 367 | if (!p2) continue; | ||
| 368 | |||
| 369 | /* decrease the counts for this symbol's tokens */ | ||
| 370 | forget_symbol(table[i].sym, len); | ||
| 371 | |||
| 372 | size = len; | ||
| 574 | 373 | ||
| 575 | do { | 374 | do { |
| 375 | *p2 = idx; | ||
| 376 | p2++; | ||
| 377 | size -= (p2 - p1); | ||
| 378 | memmove(p2, p2 + 1, size); | ||
| 379 | p1 = p2; | ||
| 380 | len--; | ||
| 381 | |||
| 382 | if (size < 2) break; | ||
| 383 | |||
| 576 | /* find the token on the symbol */ | 384 | /* find the token on the symbol */ |
| 577 | p = (unsigned char *) strstr((char *) p, (char *) str); | 385 | p2 = memmem(p1, size, str, 2); |
| 578 | if (!p) break; | ||
| 579 | |||
| 580 | if (!learn) { | ||
| 581 | /* if this symbol was used to count, decrease it */ | ||
| 582 | if (table[i].flags & SYM_FLAG_SAMPLED) | ||
| 583 | forget_symbol(table[i].sym, len); | ||
| 584 | learn = 1; | ||
| 585 | } | ||
| 586 | 386 | ||
| 587 | *p = idx; | 387 | } while (p2); |
| 588 | size = (len - (p - table[i].sym)) - tlen + 1; | ||
| 589 | memmove(p + 1, p + tlen, size); | ||
| 590 | p++; | ||
| 591 | len -= tlen - 1; | ||
| 592 | 388 | ||
| 593 | } while (size >= tlen); | 389 | table[i].len = len; |
| 594 | 390 | ||
| 595 | if(learn) { | 391 | /* increase the counts for this symbol's new tokens */ |
| 596 | table[i].len = len; | 392 | learn_symbol(table[i].sym, len); |
| 597 | /* if this symbol was used to count, learn it again */ | ||
| 598 | if(table[i].flags & SYM_FLAG_SAMPLED) | ||
| 599 | learn_symbol(table[i].sym, len); | ||
| 600 | } | ||
| 601 | } | 393 | } |
| 602 | } | 394 | } |
| 603 | 395 | ||
| 604 | /* search the token with the maximum profit */ | 396 | /* search the token with the maximum profit */ |
| 605 | static struct token *find_best_token(void) | 397 | static int find_best_token(void) |
| 606 | { | 398 | { |
| 607 | struct token *ptr,*best,*head; | 399 | int i, best, bestprofit; |
| 608 | int bestprofit; | ||
| 609 | 400 | ||
| 610 | bestprofit=-10000; | 401 | bestprofit=-10000; |
| 402 | best = 0; | ||
| 611 | 403 | ||
| 612 | /* failsafe: if the "good" list is empty search from the "bad" list */ | 404 | for (i = 0; i < 0x10000; i++) { |
| 613 | if(good_head.right == &good_head) head = &bad_head; | 405 | if (token_profit[i] > bestprofit) { |
| 614 | else head = &good_head; | 406 | best = i; |
| 615 | 407 | bestprofit = token_profit[i]; | |
| 616 | ptr = head->right; | ||
| 617 | best = NULL; | ||
| 618 | while (ptr != head) { | ||
| 619 | if (ptr->profit > bestprofit) { | ||
| 620 | bestprofit = ptr->profit; | ||
| 621 | best = ptr; | ||
| 622 | } | 408 | } |
| 623 | ptr = ptr->right; | ||
| 624 | } | 409 | } |
| 625 | |||
| 626 | return best; | 410 | return best; |
| 627 | } | 411 | } |
| 628 | 412 | ||
| 629 | /* this is the core of the algorithm: calculate the "best" table */ | 413 | /* this is the core of the algorithm: calculate the "best" table */ |
| 630 | static void optimize_result(void) | 414 | static void optimize_result(void) |
| 631 | { | 415 | { |
| 632 | struct token *best; | 416 | int i, best; |
| 633 | int i; | ||
| 634 | 417 | ||
| 635 | /* using the '\0' symbol last allows compress_symbols to use standard | 418 | /* using the '\0' symbol last allows compress_symbols to use standard |
| 636 | * fast string functions */ | 419 | * fast string functions */ |
| @@ -644,14 +427,12 @@ static void optimize_result(void) | |||
| 644 | best = find_best_token(); | 427 | best = find_best_token(); |
| 645 | 428 | ||
| 646 | /* place it in the "best" table */ | 429 | /* place it in the "best" table */ |
| 647 | best_table_len[i] = best->len; | 430 | best_table_len[i] = 2; |
| 648 | memcpy(best_table[i], best->data, best_table_len[i]); | 431 | best_table[i][0] = best & 0xFF; |
| 649 | /* zero terminate the token so that we can use strstr | 432 | best_table[i][1] = (best >> 8) & 0xFF; |
| 650 | in compress_symbols */ | ||
| 651 | best_table[i][best_table_len[i]]='\0'; | ||
| 652 | 433 | ||
| 653 | /* replace this token in all the valid symbols */ | 434 | /* replace this token in all the valid symbols */ |
| 654 | compress_symbols(best_table[i], best_table_len[i], i); | 435 | compress_symbols(best_table[i], i); |
| 655 | } | 436 | } |
| 656 | } | 437 | } |
| 657 | } | 438 | } |
| @@ -659,39 +440,28 @@ static void optimize_result(void) | |||
| 659 | /* start by placing the symbols that are actually used on the table */ | 440 | /* start by placing the symbols that are actually used on the table */ |
| 660 | static void insert_real_symbols_in_table(void) | 441 | static void insert_real_symbols_in_table(void) |
| 661 | { | 442 | { |
| 662 | int i, j, c; | 443 | unsigned int i, j, c; |
| 663 | 444 | ||
| 664 | memset(best_table, 0, sizeof(best_table)); | 445 | memset(best_table, 0, sizeof(best_table)); |
| 665 | memset(best_table_len, 0, sizeof(best_table_len)); | 446 | memset(best_table_len, 0, sizeof(best_table_len)); |
| 666 | 447 | ||
| 667 | for (i = 0; i < cnt; i++) { | 448 | for (i = 0; i < table_cnt; i++) { |
| 668 | if (table[i].flags & SYM_FLAG_VALID) { | 449 | for (j = 0; j < table[i].len; j++) { |
| 669 | for (j = 0; j < table[i].len; j++) { | 450 | c = table[i].sym[j]; |
| 670 | c = table[i].sym[j]; | 451 | best_table[c][0]=c; |
| 671 | best_table[c][0]=c; | 452 | best_table_len[c]=1; |
| 672 | best_table_len[c]=1; | ||
| 673 | } | ||
| 674 | } | 453 | } |
| 675 | } | 454 | } |
| 676 | } | 455 | } |
| 677 | 456 | ||
| 678 | static void optimize_token_table(void) | 457 | static void optimize_token_table(void) |
| 679 | { | 458 | { |
| 680 | memset(hash_table, 0, sizeof(hash_table)); | ||
| 681 | |||
| 682 | good_head.left = &good_head; | ||
| 683 | good_head.right = &good_head; | ||
| 684 | |||
| 685 | bad_head.left = &bad_head; | ||
| 686 | bad_head.right = &bad_head; | ||
| 687 | |||
| 688 | build_initial_tok_table(); | 459 | build_initial_tok_table(); |
| 689 | 460 | ||
| 690 | insert_real_symbols_in_table(); | 461 | insert_real_symbols_in_table(); |
| 691 | 462 | ||
| 692 | /* When valid symbol is not registered, exit to error */ | 463 | /* When valid symbol is not registered, exit to error */ |
| 693 | if (good_head.left == good_head.right && | 464 | if (!table_cnt) { |
| 694 | bad_head.left == bad_head.right) { | ||
| 695 | fprintf(stderr, "No valid symbol.\n"); | 465 | fprintf(stderr, "No valid symbol.\n"); |
| 696 | exit(1); | 466 | exit(1); |
| 697 | } | 467 | } |
| @@ -700,8 +470,7 @@ static void optimize_token_table(void) | |||
| 700 | } | 470 | } |
| 701 | 471 | ||
| 702 | 472 | ||
| 703 | int | 473 | int main(int argc, char **argv) |
| 704 | main(int argc, char **argv) | ||
| 705 | { | 474 | { |
| 706 | if (argc >= 2) { | 475 | if (argc >= 2) { |
| 707 | int i; | 476 | int i; |
