shtr_line_list.c (26583B)
1 /* Copyright (C) 2022, 2025, 2026 |Méso|Star> (contact@meso-star.com) 2 * Copyright (C) 2025, 2026 Université de Lorraine 3 * Copyright (C) 2022 Centre National de la Recherche Scientifique 4 * Copyright (C) 2022 Université Paul Sabatier 5 * 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation, either version 3 of the License, or 9 * (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. */ 18 19 #include "shtr_c.h" 20 #include "shtr_cache.h" 21 #include "shtr_line_list_c.h" 22 #include "shtr_param.h" 23 24 #include <rsys/cstr.h> 25 #include <rsys/text_reader.h> 26 27 /* Maximum size of a compressed block, which in the worst case could correspond 28 * to the initial block size plus an overhead of 6 bytes, in addition to 5 bytes 29 * per 16 KB of uncompressed data (see https://www.zlib.net/zlib_tech.html) */ 30 #define ZCHUNK_MAX_SIZE (CHUNK_SIZE + 6 + (5*(CHUNK_SIZE+16383/*ceil*/)/16384)) 31 32 /******************************************************************************* 33 * Compression API 34 ******************************************************************************/ 35 struct zctx { 36 struct line* lines; /* Uncompressed Lines */ 37 size_t nlines; /* Number of uncompressed lines */ 38 39 struct line last_line; /* Last line added. Used to check the order of lines */ 40 41 char* zlines; /* Compressed lines */ 42 43 z_stream stream; /* zlib */ 44 int zlib_is_init; 45 46 struct shtr* shtr; 47 }; 48 static const struct zctx ZCTX_NULL = {0}; 49 50 static voidpf 51 zalloc_func(voidpf opaque, uInt items, uInt size) 52 { 53 ASSERT(opaque); 54 return MEM_CALLOC((struct mem_allocator*)opaque, items, size); 55 } 56 57 static void 58 zfree_func(voidpf opaque, voidpf address) 59 { 60 ASSERT(opaque); 61 MEM_RM((struct mem_allocator*)opaque, address); 62 } 63 64 static void 65 zctx_release(struct zctx* zctx) 66 { 67 ASSERT(zctx); 68 if(zctx->lines) MEM_RM(zctx->shtr->allocator, zctx->lines); 69 if(zctx->zlines) MEM_RM(zctx->shtr->allocator, zctx->zlines); 70 if(zctx->zlib_is_init) deflateEnd(&zctx->stream); 71 SHTR(ref_put(zctx->shtr)); 72 } 73 74 static res_T 75 zctx_init(struct zctx* zctx, struct shtr* shtr, const int level) 76 { 77 int ret = Z_OK; 78 int z_level = 0; 79 res_T res = RES_OK; 80 ASSERT(zctx && shtr); 81 82 *zctx = ZCTX_NULL; 83 84 SHTR(ref_get(shtr)); 85 zctx->shtr = shtr; 86 zctx->nlines = 0; 87 88 /* Allocate memory of uncompressed data */ 89 zctx->lines = MEM_CALLOC 90 (zctx->shtr->allocator, NLINES_PER_CHUNK, sizeof(*zctx->lines)); 91 if(!zctx->lines) { res = RES_MEM_ERR; goto error; } 92 93 94 /* Define the zlib compression level */ 95 if(level == SHTR_DEFAULT_COMPRESSION) { 96 z_level = Z_DEFAULT_COMPRESSION; 97 } else { 98 z_level = CLAMP(level, 0, 9); /* zlib compression level in [0,9] */ 99 } 100 101 if(z_level != 0) { 102 /* Allocate memory of compressed data */ 103 zctx->zlines = MEM_ALLOC(zctx->shtr->allocator, ZCHUNK_MAX_SIZE); 104 if(!zctx->zlines) { res = RES_MEM_ERR; goto error; } 105 106 /* Initialize zlib */ 107 zctx->stream.zalloc = zalloc_func; 108 zctx->stream.zfree = zfree_func; 109 zctx->stream.opaque = zctx->shtr->allocator; 110 ret = deflateInit(&zctx->stream, z_level); 111 if(ret != Z_OK) { res = RES_UNKNOWN_ERR; goto error; } 112 zctx->zlib_is_init = 1; 113 } 114 115 exit: 116 return res; 117 error: 118 zctx_release(zctx); 119 goto exit; 120 } 121 122 static res_T 123 zctx_deflate(struct zctx* zctx, struct shtr_line_list* list) 124 { 125 struct zchunk zchunk = ZCHUNK_NULL__; 126 char* block = NULL; 127 size_t sz_total = 0; 128 size_t nblocks = 0; 129 size_t n = 0; 130 int ret = 0; 131 res_T res = RES_OK; 132 133 ASSERT(zctx && list); 134 135 if(!zctx->nlines) goto exit; /* Nothing to do */ 136 137 if(!zctx->zlib_is_init) { /* Compression is disabled */ 138 zchunk.size = (uint32_t)(zctx->nlines * sizeof(*zctx->lines)); 139 140 } else { 141 /* Setup input/output for zlib */ 142 zctx->stream.next_in = (unsigned char*)zctx->lines; 143 zctx->stream.avail_in = (uInt)(zctx->nlines * sizeof(*zctx->lines)); 144 zctx->stream.next_out = (unsigned char*)zctx->zlines; 145 zctx->stream.avail_out = ZCHUNK_MAX_SIZE; 146 147 /* Compress */ 148 ret = deflate(&zctx->stream, Z_FINISH); 149 if(ret != Z_STREAM_END) { res = RES_UNKNOWN_ERR; goto error; } 150 151 CHK(deflateReset(&zctx->stream) == Z_OK); 152 153 /* Calculate the size after compression */ 154 zchunk.size = ZCHUNK_MAX_SIZE - zctx->stream.avail_out; 155 } 156 157 /* Calculate the total size already allocated for compressed lines */ 158 nblocks = darray_charp_size_get(&list->blocks); 159 sz_total = nblocks * BLOCK_SIZE; 160 161 /* Check that the last memory block has enough space to store the compressed 162 * chunk */ 163 n = darray_zchunk_size_get(&list->zchunks); 164 if(n) { /* Is there a block? */ 165 struct zchunk* prev_chunk = &darray_zchunk_data_get(&list->zchunks)[n-1]; 166 size_t sz_in_use = prev_chunk->offset + prev_chunk->size; 167 size_t sz_remain = sz_total - sz_in_use; 168 169 if(sz_remain > zchunk.size) { 170 zchunk.offset = sz_in_use; 171 block = darray_charp_data_get(&list->blocks)[nblocks-1]; 172 } 173 } 174 175 /* No memory available. Allocate a new block */ 176 if(!block) { 177 block = MEM_CALLOC(list->shtr->allocator, 1, BLOCK_SIZE); 178 if(!block) { res = RES_MEM_ERR; goto error; } 179 180 res = darray_charp_push_back(&list->blocks, &block); 181 if(res != RES_OK) goto error; 182 183 zchunk.offset = sz_total; 184 } 185 186 /* Register the chunk */ 187 res = darray_zchunk_push_back(&list->zchunks, &zchunk); 188 if(res != RES_OK) goto error; 189 190 if(zctx->zlib_is_init) { 191 /* Save compressed chunk data */ 192 memcpy(block + zchunk.offset % BLOCK_SIZE, zctx->zlines, zchunk.size); 193 } else { 194 /* Save un-compressed chunk data */ 195 memcpy(block + zchunk.offset % BLOCK_SIZE, zctx->lines, zchunk.size); 196 } 197 198 /* Update the number of fully recorded lines, 199 * i.e., compressed and stored in the list */ 200 list->nlines += zctx->nlines; 201 202 /* No lines waiting for compression. */ 203 zctx->nlines = 0; 204 205 exit: 206 return res; 207 error: 208 ERROR(list->shtr, "Error while compressing lines -- %s\n", 209 zctx->stream.msg ? zctx->stream.msg : res_to_cstr(res)); 210 goto exit; 211 } 212 213 /******************************************************************************* 214 * Helper functions 215 ******************************************************************************/ 216 static res_T 217 check_shtr_line_list_load_args(const struct shtr_line_list_load_args* args) 218 { 219 if(!args) return RES_BAD_ARG; 220 221 /* Source is missing */ 222 if(!args->file && !args->filename) return RES_BAD_ARG; 223 224 return RES_OK; 225 } 226 227 static res_T 228 create_line_list 229 (struct shtr* shtr, 230 struct shtr_line_list** out_list) 231 { 232 struct shtr_line_list* list = NULL; 233 res_T res = RES_OK; 234 ASSERT(shtr && out_list); 235 236 list = MEM_CALLOC(shtr->allocator, 1, sizeof(*list)); 237 if(!list) { 238 ERROR(shtr, "Could not allocate the list of lines.\n"); 239 res = RES_MEM_ERR; 240 goto error; 241 } 242 ref_init(&list->ref); 243 SHTR(ref_get(shtr)); 244 list->shtr = shtr; 245 darray_zchunk_init(shtr->allocator, &list->zchunks); 246 darray_charp_init(shtr->allocator, &list->blocks); 247 list->info = SHTR_LINE_LIST_INFO_NULL; 248 249 res = cache_create(shtr, &list->cache); 250 if(res != RES_OK) goto error; 251 252 exit: 253 *out_list = list; 254 return res; 255 error: 256 if(list) { 257 SHTR(line_list_ref_put(list)); 258 list = NULL; 259 } 260 goto exit; 261 } 262 263 static res_T 264 setup_zlib(struct shtr_line_list* list) 265 { 266 int ret = Z_OK; /* zlib */ 267 res_T res = RES_OK; 268 ASSERT(list); 269 270 list->z_stream.zalloc = zalloc_func; 271 list->z_stream.zfree = zfree_func; 272 list->z_stream.opaque = list->shtr->allocator; 273 ret = inflateInit(&list->z_stream); 274 if(ret != Z_OK) { res = RES_UNKNOWN_ERR; goto error; } 275 276 list->zlib_is_init = 1; 277 278 exit: 279 return res; 280 error: 281 ERROR(list->shtr, 282 "Error intializing line decompressor -- %s\n", res_to_cstr(res)); 283 goto exit; 284 } 285 286 static void 287 line_encode(const struct shtr_line* line, struct line* line_encoded) 288 { 289 union { float flt; int32_t i32; } ucast; 290 ASSERT(line && line_encoded); 291 292 /* Keep the wavenumber and the intensity as it */ 293 line_encoded->wavenumber = line->wavenumber; 294 line_encoded->intensity = line->intensity; 295 296 /* Encode the lower state energy and delta air in single precision */ 297 line_encoded->lower_state_energy = (float)line->lower_state_energy; 298 line_encoded->delta_air = (float)line->delta_air; 299 300 /* Store gamma_air as an unsigned fixed-point number (0:14), i.e., 0 bit for 301 * the integer part and 14 bits for the fractional part */ 302 ASSERT((int64_t)line->gamma_air == 0 && line->gamma_air >= 0); 303 line_encoded->gair14_gself14_isoid4 = 304 ((int32_t)float2fix(line->gamma_air, 0, 14) & (BIT_I32(14)-1)) << 18; 305 306 /* Store gamma_self as an unsigned fixed-point number (0:14), i.e., 0 bit for 307 * the integer part and 14 bits for the fractional part */ 308 ASSERT((int64_t)line->gamma_self == 0 && line->gamma_self >= 0); 309 line_encoded->gair14_gself14_isoid4 |= 310 ((int32_t)float2fix(line->gamma_self, 0, 14) & (BIT_I32(14)-1)) << 4; 311 312 /* Store the isotope id on 4 bits */ 313 ASSERT(line->isotope_id_local < 16); 314 line_encoded->gair14_gself14_isoid4 |= line->isotope_id_local & (BIT_I32(4)-1); 315 316 /* Encode n_air in single precision with its last 7 bits of matissa disabled 317 * in order to store the molecule identifier */ 318 ucast.flt = (float)line->n_air; 319 ucast.i32 &= ~(BIT_I32(7)-1); 320 321 /* Store the molecule id on 7 bits */ 322 ASSERT(line->molecule_id < 128); 323 ucast.i32 |= line->molecule_id & (BIT_I32(7)-1); 324 line_encoded->nair25_molid7 = ucast.i32; 325 } 326 327 static void 328 line_decode(const struct line* line_encoded, struct shtr_line* line) 329 { 330 union { float flt; int32_t i32; } ucast; 331 int64_t i64 = 0; 332 ASSERT(line && line_encoded); 333 334 line->wavenumber = line_encoded->wavenumber; 335 line->intensity = line_encoded->intensity; 336 line->lower_state_energy = line_encoded->lower_state_energy; 337 line->delta_air = line_encoded->delta_air; 338 339 /* Convert gamma_air and gamma_self from fixed-point numbers (0:14) to 340 * double-precision floating-point numbers */ 341 i64 = (line_encoded->gair14_gself14_isoid4 >> 18) & (BIT_I32(14)-1); 342 line->gamma_air = fix2float(i64, 0, 14); 343 i64 = (line_encoded->gair14_gself14_isoid4 >> 4) & (BIT_I32(14)-1); 344 line->gamma_self = fix2float(i64, 0, 14); 345 346 /* Unpack the isotope ID */ 347 line->isotope_id_local = line_encoded->gair14_gself14_isoid4 & (BIT_I32(4)-1); 348 349 /* Unpack the molecule ID */ 350 ucast.i32 = line_encoded->nair25_molid7; 351 line->molecule_id = ucast.i32 & (BIT_I32(7)-1); 352 ucast.i32 &= ~(BIT_I32(7)-1); 353 354 /* Convert n_air from single precision to double precision */ 355 line->n_air = ucast.flt; 356 } 357 358 static res_T 359 register_line 360 (struct shtr_line_list* list, 361 const struct txtrdr* txtrdr, 362 const struct shtr_line* line, 363 struct zctx* zctx) 364 { 365 struct shtr_line ln = SHTR_LINE_NULL; 366 struct line ln_encoded = LINE_NULL; 367 res_T res = RES_OK; 368 369 /* Pre-conditions */ 370 ASSERT(list && txtrdr && line); 371 ASSERT(zctx && zctx->nlines < NLINES_PER_CHUNK); 372 373 line_encode(line, &ln_encoded); 374 375 /* Check if a line has been saved. If so, ensure that the lines are sorted */ 376 if(darray_zchunk_size_get(&list->zchunks) || zctx->nlines) { 377 if(zctx->last_line.wavenumber > ln_encoded.wavenumber) { 378 ERROR(list->shtr, 379 "%s:%lu: lines are not sorted in ascending order wrt their wavenumber.\n", 380 txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr)); 381 res = RES_BAD_ARG; 382 goto error; 383 } 384 } 385 386 zctx->last_line = ln_encoded; 387 zctx->lines[zctx->nlines] = ln_encoded; 388 zctx->nlines += 1; 389 390 /* The chunk is full. Compress it */ 391 if(zctx->nlines == NLINES_PER_CHUNK) { 392 res = zctx_deflate(zctx, list); 393 if(res != RES_OK) goto error; 394 } 395 396 line_decode(&ln_encoded, &ln); 397 ASSERT(ln.molecule_id == line->molecule_id); 398 ASSERT(ln.isotope_id_local == line->isotope_id_local); 399 400 #define UPDATE_INFO(Name) { \ 401 const double err = fabs(line->Name - ln.Name); \ 402 list->info.Name.range[0] = MMIN(list->info.Name.range[0], ln.Name); \ 403 list->info.Name.range[1] = MMAX(list->info.Name.range[1], ln.Name); \ 404 list->info.Name.err = MMAX(list->info.Name.err, err); \ 405 } (void) 0 406 UPDATE_INFO(wavenumber); 407 UPDATE_INFO(intensity); 408 UPDATE_INFO(gamma_air); 409 UPDATE_INFO(gamma_self); 410 UPDATE_INFO(lower_state_energy); 411 UPDATE_INFO(n_air); 412 UPDATE_INFO(delta_air); 413 #undef UPDATE_INFO 414 415 exit: 416 return res; 417 error: 418 goto exit; 419 } 420 421 static res_T 422 parse_line 423 (struct shtr_line_list* list, 424 struct txtrdr* txtrdr, 425 struct shtr_line* ln) 426 { 427 struct param_desc param = PARAM_DESC_NULL; 428 struct shtr* shtr = NULL; 429 char* line = NULL; 430 char* str = NULL; 431 char* end = NULL; 432 char backup; 433 int molecule_id; 434 int isotope_id_local; 435 res_T res = RES_OK; 436 437 ASSERT(list && txtrdr && ln); 438 439 line = txtrdr_get_line(txtrdr); 440 ASSERT(line); 441 442 shtr = list->shtr; 443 param.path = txtrdr_get_name(txtrdr); 444 param.line = txtrdr_get_line_num(txtrdr); 445 446 str = end = line; 447 backup = str[0]; 448 #define NEXT(Size) { \ 449 *end = backup; \ 450 str = end; \ 451 end = str+(Size); \ 452 backup = *end; \ 453 *end = '\0'; \ 454 } (void)0 455 #define PARSE(Var, Size, Type, Name, Low, Upp, LowIncl, UppIncl) { \ 456 NEXT(Size); \ 457 param.name = (Name); \ 458 param.low = (Low); \ 459 param.upp = (Upp); \ 460 param.is_low_incl = (LowIncl); \ 461 param.is_upp_incl = (UppIncl); \ 462 res = parse_param_##Type(shtr, str, ¶m, Var); \ 463 if(res != RES_OK) goto error; \ 464 } (void)0 465 466 PARSE(&molecule_id, 2, int, "molecule identifier", 0,99,1,1); 467 ln->molecule_id = (int32_t)molecule_id; 468 469 PARSE(&isotope_id_local, 1, int, "isotope local identifier", 0,9,1,1); 470 ln->isotope_id_local = (int32_t) 471 (isotope_id_local == 0 ? 9 : (isotope_id_local - 1)); 472 473 PARSE(&ln->wavenumber, 12, double, "central wavenumber", 0,INF,0,1); 474 PARSE(&ln->intensity, 10, double, "reference intensity", 0,INF,0,1); 475 476 NEXT(10); /* Skip the Enstein coef */ 477 478 PARSE(&ln->gamma_air, 5, double, "air broadening half-width", 0,INF,1,1); 479 PARSE(&ln->gamma_self, 5, double, "self broadening half-width", 0,INF,1,1); 480 481 /* Handle unavailable lower state energy */ 482 PARSE(&ln->lower_state_energy, 10, double, "lower state energy",-INF,INF,1,1); 483 if(ln->lower_state_energy == -1) { 484 WARN(shtr, 485 "%s:%lu: the lower state energy is unavailable for this line, so it is " 486 "ignored.\n", txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr)); 487 goto exit; /* Skip the line */ 488 } 489 /* Check the domain validity */ 490 if(ln->lower_state_energy < 0) { 491 ERROR(shtr, 492 "%s:%lu: invalid lower state energy %g. It must be in [0, INF].\n", 493 txtrdr_get_name(txtrdr), txtrdr_get_line_num(txtrdr), 494 ln->lower_state_energy); 495 res = RES_BAD_ARG; 496 goto error; 497 } 498 499 PARSE(&ln->n_air, 4, double, "temperature-dependent exponent",-INF,INF,1,1); 500 PARSE(&ln->delta_air, 8, double, "air-pressure wavenumber shift", -INF,INF,1,1); 501 502 /* Skip the remaining values */ 503 504 #undef NEXT 505 #undef PARSE 506 507 /* Check the size of the remaining data to ensure that there is at least the 508 * expected number of bytes wrt the HITRAN fileformat */ 509 *end = backup; 510 str = end; 511 if(strlen(str) != 93) { 512 ERROR(list->shtr, "%s:%lu: missing data after delta air.\n", 513 param.path, (unsigned long)param.line); 514 res = RES_BAD_ARG; 515 goto error; 516 } 517 518 exit: 519 return res; 520 error: 521 goto exit; 522 } 523 524 static res_T 525 load_stream 526 (struct shtr* shtr, 527 FILE* stream, 528 const struct shtr_line_list_load_args* args, 529 struct shtr_line_list** out_lines) 530 { 531 struct zctx zctx = ZCTX_NULL; 532 struct shtr_line_list* list = NULL; 533 struct txtrdr* txtrdr = NULL; 534 const char* name = NULL; 535 res_T res = RES_OK; 536 537 ASSERT(shtr && stream && args && out_lines); 538 539 if(args->file) { /* Load from stream */ 540 name = args->filename ? args->filename : "<stream>"; 541 } else { 542 name = args->filename; 543 } 544 545 res = create_line_list(shtr, &list); 546 if(res != RES_OK) goto error; 547 548 if(args->compression_level > 0) { 549 res = setup_zlib(list); 550 if(res != RES_OK) goto error; 551 } 552 553 res = zctx_init(&zctx, shtr, args->compression_level); 554 if(res != RES_OK) goto error; 555 556 res = txtrdr_stream(list->shtr->allocator, stream, name, 557 0/*No comment char*/, &txtrdr); 558 if(res != RES_OK) { 559 ERROR(shtr, "%s: error creating the text reader -- %s.\n", 560 name, res_to_cstr(res)); 561 goto error; 562 } 563 564 for(;;) { 565 struct shtr_line ln = SHTR_LINE_NULL; 566 567 res = txtrdr_read_line(txtrdr); 568 if(res != RES_OK) { 569 ERROR(shtr, "%s: error reading the line `%lu' -- %s.\n", 570 name, (unsigned long)txtrdr_get_line_num(txtrdr), res_to_cstr(res)); 571 goto error; 572 } 573 574 if(!txtrdr_get_cline(txtrdr)) break; /* No more parsed line */ 575 576 res = parse_line(list, txtrdr, &ln); 577 if(res != RES_OK) goto error; 578 579 res = register_line(list, txtrdr, &ln, &zctx); 580 if(res != RES_OK) goto error; 581 } 582 583 /* Ensure that remaining lines are compressed and stored */ 584 res = zctx_deflate(&zctx, list); 585 if(res != RES_OK) goto error; 586 587 exit: 588 if(txtrdr) txtrdr_ref_put(txtrdr); 589 zctx_release(&zctx); 590 *out_lines = list; 591 return res; 592 error: 593 if(list) { 594 SHTR(line_list_ref_put(list)); 595 list = NULL; 596 } 597 goto exit; 598 } 599 600 static res_T 601 decompress_zchunk 602 (struct shtr_line_list* list, 603 const size_t chunk_id, 604 struct line lines[NLINES_PER_CHUNK]) 605 { 606 const struct zchunk* zchunk = NULL; 607 char* block = NULL; 608 size_t block_id = 0; 609 size_t block_offset = 0; 610 int ret = Z_OK; /* zlib */ 611 res_T res = RES_OK; 612 613 ASSERT(list && lines && chunk_id < darray_zchunk_size_get(&list->zchunks)); 614 615 zchunk = darray_zchunk_cdata_get(&list->zchunks) + chunk_id; 616 block_id = zchunk->offset / BLOCK_SIZE; 617 block_offset = zchunk->offset % BLOCK_SIZE; 618 619 block = darray_charp_cdata_get(&list->blocks)[block_id]; 620 621 if(!list->zlib_is_init) { 622 /* Data are not compressed */ 623 memcpy(lines, block+block_offset, zchunk->size); 624 625 } else { 626 list->z_stream.next_in = (unsigned char*)(block + block_offset); 627 list->z_stream.avail_in = (uInt)zchunk->size; 628 list->z_stream.next_out = (unsigned char*)lines; 629 list->z_stream.avail_out = (uInt)(sizeof(struct line)*NLINES_PER_CHUNK); 630 ret = inflate(&list->z_stream, Z_FINISH); 631 if(ret != Z_STREAM_END) { 632 ASSERT(list->z_stream.msg); 633 ERROR(list->shtr, "Error decompressing the chunk of lines -- %s\n", 634 list->z_stream.msg); 635 res = RES_UNKNOWN_ERR; 636 goto error; 637 } 638 639 CHK(inflateReset(&list->z_stream) == Z_OK); 640 } 641 642 exit: 643 return res; 644 error: 645 goto exit; 646 } 647 648 static void 649 release_lines(ref_T * ref) 650 { 651 struct shtr* shtr = NULL; 652 struct shtr_line_list* list = CONTAINER_OF(ref, struct shtr_line_list, ref); 653 char** blocks = NULL; 654 size_t i=0, n=0; 655 656 ASSERT(ref); 657 658 shtr = list->shtr; 659 660 if(list->cache) cache_ref_put(list->cache); 661 if(list->zlib_is_init) inflateEnd(&list->z_stream); 662 663 n = darray_charp_size_get(&list->blocks); 664 blocks = darray_charp_data_get(&list->blocks); 665 FOR_EACH(i, 0, n) { if(blocks[i]) MEM_RM(shtr->allocator, blocks[i]); } 666 667 darray_zchunk_release(&list->zchunks); 668 darray_charp_release(&list->blocks); 669 MEM_RM(shtr->allocator, list); 670 SHTR(ref_put(shtr)); 671 } 672 673 /******************************************************************************* 674 * Exported functions 675 ******************************************************************************/ 676 res_T 677 shtr_line_list_load 678 (struct shtr* shtr, 679 const struct shtr_line_list_load_args* args, 680 struct shtr_line_list** list) 681 { 682 FILE* file = NULL; 683 res_T res = RES_OK; 684 685 if(!shtr || !list) { res = RES_BAD_ARG; goto error; } 686 res = check_shtr_line_list_load_args(args); 687 if(res != RES_OK) goto error; 688 689 if(args->file) { /* Load from stream */ 690 file = args->file; 691 692 } else { /* Load from file */ 693 file = fopen(args->filename, "r"); 694 if(!file) { 695 ERROR(shtr, "%s: error opening file `%s'.\n", FUNC_NAME, args->filename); 696 res = RES_IO_ERR; 697 goto error; 698 } 699 } 700 701 res = load_stream(shtr, file, args, list); 702 if(res != RES_OK) goto error; 703 704 exit: 705 if(file && file != args->file) fclose(file); 706 return res; 707 error: 708 goto exit; 709 } 710 711 res_T 712 shtr_line_list_create_from_stream 713 (struct shtr* shtr, 714 FILE* stream, 715 struct shtr_line_list** out_list) 716 { 717 struct shtr_line_list* list = NULL; 718 char** blocks = NULL; 719 size_t i=0, n=0; 720 int is_compression_enabled = 0; 721 int version = 0; 722 res_T res = RES_OK; 723 724 if(!shtr || !out_list || !stream) { 725 res = RES_BAD_ARG; 726 goto error; 727 } 728 729 res = create_line_list(shtr, &list); 730 if(res != RES_OK) goto error; 731 732 #define READ(Var, Nb) { \ 733 if(fread((Var), sizeof(*(Var)), (Nb), stream) != (Nb)) { \ 734 if(feof(stream)) { \ 735 res = RES_BAD_ARG; \ 736 } else if(ferror(stream)) { \ 737 res = RES_IO_ERR; \ 738 } else { \ 739 res = RES_UNKNOWN_ERR; \ 740 } \ 741 ERROR(shtr, \ 742 "%s: error reading line list -- %s.\n", FUNC_NAME, res_to_cstr(res)); \ 743 goto error; \ 744 } \ 745 } (void)0 746 747 READ(&version, 1); 748 if(version != SHTR_LINE_LIST_VERSION) { 749 ERROR(shtr, 750 "%s: unexpected line list version %d. " 751 "Expecting a line list in version %d.\n", 752 FUNC_NAME, version, SHTR_LINE_LIST_VERSION); 753 res = RES_BAD_ARG; 754 goto error; 755 } 756 757 READ(&list->nlines, 1); 758 READ(&is_compression_enabled, 1); 759 760 /* Memory descriptor of compressed chunks */ 761 READ(&n, 1); 762 if((res = darray_zchunk_resize(&list->zchunks, n)) != RES_OK) goto error; 763 READ(darray_zchunk_data_get(&list->zchunks), n); 764 765 /* Compressed data stored in memory blocks */ 766 READ(&n, 1); 767 if((res = darray_charp_resize(&list->blocks, n)) != RES_OK) goto error; 768 blocks = darray_charp_data_get(&list->blocks); 769 FOR_EACH(i, 0, n) { 770 blocks[i] = MEM_ALLOC(list->shtr->allocator, BLOCK_SIZE); 771 if(!blocks[i]) { res = RES_MEM_ERR; goto error; } 772 READ(blocks[i], BLOCK_SIZE); 773 } 774 775 /* Informations on line parameters */ 776 READ(&list->info, 1); 777 778 #undef READ 779 780 if(is_compression_enabled) { 781 res = setup_zlib(list); 782 if(res != RES_OK) goto error; 783 } 784 785 exit: 786 if(out_list) *out_list = list; 787 return res; 788 error: 789 if(list) { SHTR(line_list_ref_put(list)); list = NULL; } 790 goto exit; 791 } 792 793 res_T 794 shtr_line_list_ref_get(struct shtr_line_list* list) 795 { 796 if(!list) return RES_BAD_ARG; 797 ref_get(&list->ref); 798 return RES_OK; 799 } 800 801 res_T 802 shtr_line_list_ref_put(struct shtr_line_list* list) 803 { 804 if(!list) return RES_BAD_ARG; 805 ref_put(&list->ref, release_lines); 806 return RES_OK; 807 } 808 809 res_T 810 shtr_line_list_get_size 811 (const struct shtr_line_list* list, 812 size_t* nlines) 813 { 814 if(!list || !nlines) return RES_BAD_ARG; 815 *nlines = list->nlines; 816 return RES_OK; 817 } 818 819 res_T 820 shtr_line_list_at 821 (struct shtr_line_list* list, 822 const size_t i, 823 struct shtr_line* line) 824 { 825 struct line ln_encoded = LINE_NULL; 826 res_T res = RES_OK; 827 828 if(!list || !line || i >= list->nlines) return RES_BAD_ARG; 829 830 res = cache_get_line(list->cache, i, &ln_encoded); 831 832 if(res != RES_OK) { /* Cache miss */ 833 const size_t chunk_id = i / NLINES_PER_CHUNK; 834 const size_t line_id = i % NLINES_PER_CHUNK; 835 struct line lines[NLINES_PER_CHUNK]; 836 837 if((res = decompress_zchunk(list, chunk_id, lines)) != RES_OK) goto error; 838 cache_put_chunk(list->cache, chunk_id, lines); 839 840 ln_encoded = lines[line_id]; 841 } 842 843 line_decode(&ln_encoded, line); 844 845 exit: 846 return res; 847 error: 848 goto exit; 849 } 850 851 res_T 852 shtr_line_list_write 853 (const struct shtr_line_list* list, 854 FILE* stream) 855 { 856 char* const* blocks = NULL; 857 size_t i=0, n=0; 858 res_T res = RES_OK; 859 860 if(!list || !stream) { res = RES_BAD_ARG; goto error; } 861 862 #define WRITE(Var, Nb) { \ 863 if(fwrite((Var), sizeof(*(Var)), (Nb), stream) != (Nb)) { \ 864 res = RES_IO_ERR; \ 865 ERROR(list->shtr, \ 866 "%s: error writing line list -- %s\n", FUNC_NAME, res_to_cstr(res)); \ 867 goto error; \ 868 } \ 869 } (void)0 870 871 /* Version management */ 872 WRITE(&SHTR_LINE_LIST_VERSION, 1); 873 874 /* Number of lines in the list */ 875 WRITE(&list->nlines, 1); 876 877 /* Is decompression enabled */ 878 WRITE(&list->zlib_is_init, 1); 879 880 /* Memory descriptor of compressed chunks */ 881 n = darray_zchunk_size_get(&list->zchunks); 882 WRITE(&n, 1); 883 WRITE(darray_zchunk_cdata_get(&list->zchunks), n); 884 885 /* Compressed data stored in memory blocks */ 886 blocks = darray_charp_cdata_get(&list->blocks); 887 n = darray_charp_size_get(&list->blocks); 888 WRITE(&n, 1); 889 FOR_EACH(i, 0, n) { WRITE(blocks[i], BLOCK_SIZE); } 890 891 /* Informations on line parameters */ 892 WRITE(&list->info, 1); 893 894 #undef WRITE 895 896 exit: 897 return res; 898 error: 899 goto exit; 900 } 901 902 res_T 903 shtr_line_list_get_info 904 (const struct shtr_line_list* list, 905 struct shtr_line_list_info* info) 906 { 907 if(!list || !info) return RES_BAD_ARG; 908 *info = list->info; 909 return RES_OK; 910 }