Freeciv-3.3
Loading...
Searching...
No Matches
ioz.c
Go to the documentation of this file.
1/***********************************************************************
2 Freeciv - Copyright (C) 1996 - A Kjeldberg, L Gregersen, P Unold
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License as published by
5 the Free Software Foundation; either version 2, or (at your option)
6 any later version.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 GNU General Public License for more details.
12***********************************************************************/
13
14/***********************************************************************
15 An IO layer to support transparent compression/uncompression.
16 (Currently only "required" functionality is supported.)
17
18 There are various reasons for making this a full-blown module
19 instead of just defining a few macros:
20
21 - Ability to switch between compressed and uncompressed at run-time
22 (zlib with compression level 0 saves uncompressed, but still with
23 gzip header, so non-zlib server cannot read the savefile).
24
25 - Flexibility to add other methods if desired (eg, bzip2, arbitrary
26 external filter program, etc).
27
28 FIXME: When zlib support _not_ included, should sanity check whether
29 the first few bytes are gzip marker and complain if so.
30***********************************************************************/
31
32#ifdef HAVE_CONFIG_H
33#include <fc_config.h>
34#endif
35
36#include "fc_prehdrs.h"
37
38#include <errno.h>
39#include <stdarg.h>
40#include <stdio.h>
41#include <string.h>
42
43#ifdef FREECIV_HAVE_LIBZ
44#include <zlib.h>
45#endif
46
47#ifdef HAVE_BZLIB_H
48#include <bzlib.h>
49#endif
50
51#ifdef HAVE_LZMA_H
52#include <lzma.h>
53#endif
54
55#ifdef HAVE_ZSTD_H
56#include <zstd.h>
57#endif
58
59/* utility */
60#include "log.h"
61#include "mem.h"
62#include "shared.h"
63#include "support.h"
64
65#include "ioz.h"
66
67/* If you increase size of this, be sure to increase also
68 * VSNP_BUF_SIZE in support.c so that this still fits in. */
69#define PLAIN_FILE_BUF_SIZE (8096*1024) // 8096kb
70
71#ifdef FREECIV_HAVE_LIBBZ2
72struct bzip2_struct {
73 BZFILE *file;
74 FILE *plain;
75 int error;
76 int firstbyte;
77 bool eof;
78};
79#endif /* FREECIV_HAVE_LIBBZ2 */
80
81#ifdef FREECIV_HAVE_LIBLZMA
82
83#define PLAIN_FILE_BUF_SIZE_XZ PLAIN_FILE_BUF_SIZE
84#define XZ_DECODER_TEST_SIZE (4*1024) /* 4kb */
85
86/* In my tests 7Mb proved to be not enough and with 10Mb decompression
87 succeeded in typical case. */
88#define XZ_DECODER_MEMLIMIT (65*1024*1024) /* 65Mb */
89#define XZ_DECODER_MEMLIMIT_STEP (25*1024*1024) /* Increase 25Mb at a time */
90#define XZ_DECODER_MEMLIMIT_FINAL (100*1024*1024) /* 100Mb */
91
92struct xz_struct {
94 int out_index;
96
97 /* liblzma bug workaround. This is what stream.avail_out should be,
98 calculated correctly. Used only when reading file. */
99 int out_avail;
100 int total_read;
101
102 FILE *plain;
103 uint8_t *in_buf; /* uint8_t is what xz headers use */
105 lzma_ret error;
106
107 /* There seems to be bug in liblzma decompression that if all the
108 processing happened when lzma_code() last time was called with
109 LZMA_RUN action, it does not set next_out or avail_out variables
110 to sane values when lzma_code() is called with LZMA_FINISH.
111 We never call lzma_code() with LZMA_RUN action with all the input
112 we have left. This hack_byte is kept in storage in case there is no
113 more input when we try to get it. This byte can then be provided to
114 lzma_code(LZMA_FINISH) */
115 char hack_byte;
116 bool hack_byte_used;
117};
118
120static void xz_action(fz_FILE *fp, lzma_action action);
121
122#endif /* FREECIV_HAVE_LIBLZMA */
123
124#ifdef FREECIV_HAVE_LIBZSTD
125
126#define PLAIN_FILE_BUF_SIZE_ZSTD PLAIN_FILE_BUF_SIZE
127
128struct zstd_struct {
131 FILE *plain;
132 int out_fd;
135 char *nonconst_in;
136 size_t outbuf_pos;
137 size_t error;
138};
139
140#endif /* FREECIV_HAVE_LIBZSTD */
141
144 char *buffer;
145 int pos;
146 int size;
147};
148
149struct fz_FILE_s {
151 char mode;
152 bool memory;
153 union {
155 FILE *plain; /* FZ_PLAIN */
156#ifdef FREECIV_HAVE_LIBZ
157 gzFile zlib; /* FZ_ZLIB */
158#endif
159#ifdef FREECIV_HAVE_LIBBZ2
160 struct bzip2_struct bz2;
161#endif
162#ifdef FREECIV_HAVE_LIBLZMA
163 struct xz_struct xz;
164#endif
165#ifdef FREECIV_HAVE_LIBZSTD
166 struct zstd_struct zstd;
167#endif
168 } u;
169};
170
171/************************************************************************/
174static inline bool fz_method_is_valid(enum fz_method method)
175{
176 switch (method) {
177 case FZ_PLAIN:
178#ifdef FREECIV_HAVE_LIBZ
179 case FZ_ZLIB:
180#endif
181#ifdef FREECIV_HAVE_LIBBZ2
182 case FZ_BZIP2:
183#endif
184#ifdef FREECIV_HAVE_LIBLZMA
185 case FZ_XZ:
186#endif
187#ifdef FREECIV_HAVE_LIBZSTD
188 case FZ_ZSTD:
189#endif
190 return TRUE;
191 }
192 return FALSE;
193}
194
195#define fz_method_validate(method) \
196 (fz_method_is_valid(method) ? method \
197 : (fc_assert_msg(fz_method_is_valid(method), \
198 "Unsupported compress method %d, reverting to plain.",\
199 method), FZ_PLAIN))
200
201
202/************************************************************************/
207fz_FILE *fz_from_memory(char *buffer, int size, bool control)
208{
209 fz_FILE *fp;
210
211 fp = (fz_FILE *)fc_malloc(sizeof(*fp));
212 fp->memory = TRUE;
213 fp->u.mem.control = control;
214 fp->u.mem.buffer = buffer;
215 fp->u.mem.pos = 0;
216 fp->u.mem.size = size;
217
218 return fp;
219}
220
221/************************************************************************/
230fz_FILE *fz_from_file(const char *filename, const char *in_mode,
231 enum fz_method method, int compress_level)
232{
233 fz_FILE *fp;
234 char mode[64];
235
236 if (!is_reg_file_for_access(filename, in_mode[0] == 'w')) {
237 return nullptr;
238 }
239
240 fp = (fz_FILE *)fc_malloc(sizeof(*fp));
241 fp->memory = FALSE;
242 sz_strlcpy(mode, in_mode);
243
244 if (mode[0] == 'w') {
245 /* Writing: */
246 fp->mode = 'w';
247 } else {
248#if defined(FREECIV_HAVE_LIBBZ2) || defined(FREECIV_HAVE_LIBLZMA) || defined (FREECIV_HAVE_LIBZSTD)
249 char test_mode[4];
250
251 sz_strlcpy(test_mode, mode);
252 sz_strlcat(test_mode, "b");
253#endif /* FREECIV_HAVE_LIBBZ2 || FREECIV_HAVE_LIBLZMA || FREECIV_HAVE_LIBZSTD */
254
255 /* Reading: ignore specified method and try each: */
256 fp->mode = 'r';
257
258#ifdef FREECIV_HAVE_LIBBZ2
259 /* Try to open as bzip2 file
260 This is simplest test, so do it first. */
261 fp->u.bz2.plain = fc_fopen(filename, test_mode);
262 if (fp->u.bz2.plain) {
263 fp->u.bz2.file = BZ2_bzReadOpen(&fp->u.bz2.error, fp->u.bz2.plain, 1, 0,
264 nullptr, 0);
265 } else {
266 /* This may currently have garbage value assigned via other union member. */
267 fp->u.bz2.file = nullptr;
268 }
269 if (!fp->u.bz2.file) {
270 if (fp->u.bz2.plain) {
271 fclose(fp->u.bz2.plain);
272 }
273 free(fp);
274 return nullptr;
275 } else {
276 /* Try to read first byte out of stream so we can figure out if this
277 really is bzip2 file or not. Store byte for later use */
278 char tmp;
279 int read_len;
280
281 /* We put error to tmp variable when we don't want to overwrite
282 * error already in fp->u.bz2.error. So calls to fz_ferror() or
283 * fz_strerror() will later return what originally went wrong,
284 * and not what happened in error recovery. */
285 int tmp_err;
286
287 read_len = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, &tmp, 1);
288 if (fp->u.bz2.error != BZ_DATA_ERROR_MAGIC) {
289 /* bzip2 file */
290 if (fp->u.bz2.error == BZ_STREAM_END) {
291 /* We already reached end of file with our read of one byte */
292 if (read_len == 0) {
293 /* 0 byte file */
294 fp->u.bz2.firstbyte = -1;
295 } else {
296 fp->u.bz2.firstbyte = tmp;
297 }
298 fp->u.bz2.eof = TRUE;
299 } else if (fp->u.bz2.error != BZ_OK) {
300 /* Read failed */
301 BZ2_bzReadClose(&tmp_err, fp->u.bz2.file);
302 fclose(fp->u.bz2.plain);
303 free(fp);
304 return nullptr;
305 } else {
306 /* Read success and we can continue reading */
307 fp->u.bz2.firstbyte = tmp;
308 fp->u.bz2.eof = FALSE;
309 }
310 fp->method = FZ_BZIP2;
311 return fp;
312 }
313
314 /* Not bzip2 file */
315 BZ2_bzReadClose(&tmp_err, fp->u.bz2.file);
316 fclose(fp->u.bz2.plain);
317 }
318#endif /* FREECIV_HAVE_LIBBZ2 */
319
320#ifdef FREECIV_HAVE_LIBLZMA
321 /* Try to open as xz file */
322 fp->u.xz.memlimit = XZ_DECODER_MEMLIMIT;
323 memset(&fp->u.xz.stream, 0, sizeof(lzma_stream));
324 fp->u.xz.error = lzma_stream_decoder(&fp->u.xz.stream,
325 fp->u.xz.memlimit,
327 if (fp->u.xz.error != LZMA_OK) {
328 free(fp);
329 return nullptr;
330 }
331 fp->u.xz.plain = fc_fopen(filename, test_mode);
332 if (fp->u.xz.plain) {
333 size_t len = 0;
334
335 fp->u.xz.in_buf = fc_malloc(PLAIN_FILE_BUF_SIZE_XZ);
336
337 len = fread(fp->u.xz.in_buf, 1, XZ_DECODER_TEST_SIZE,
338 fp->u.xz.plain);
339 if (len > 0) {
341
342 fp->u.xz.stream.next_in = fp->u.xz.in_buf;
343 fp->u.xz.stream.avail_in = len;
344 fp->u.xz.out_buf = fc_malloc(PLAIN_FILE_BUF_SIZE_XZ);
345 fp->u.xz.stream.next_out = fp->u.xz.out_buf;
346 fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE_XZ;
347 len = fread(&fp->u.xz.hack_byte, 1, 1, fp->u.xz.plain);
348 if (len > 0) {
349 fp->u.xz.hack_byte_used = TRUE;
351 } else {
352 fp->u.xz.hack_byte_used = FALSE;
354 }
355 xz_action(fp, action);
356 if (fp->u.xz.error == LZMA_OK || fp->u.xz.error == LZMA_STREAM_END) {
357 fp->method = FZ_XZ;
358 fp->u.xz.out_index = 0;
359 fp->u.xz.total_read = 0;
360 fp->u.xz.out_avail = fp->u.xz.stream.total_out;
361 return fp;
362 }
363
364 free(fp->u.xz.out_buf);
365 }
366 fclose(fp->u.xz.plain);
367 lzma_end(&fp->u.xz.stream);
368 free(fp->u.xz.in_buf);
369 } else {
370 free(fp);
371 return nullptr;
372 }
373#endif /* FREECIV_HAVE_LIBLZMA */
374
375#ifdef FREECIV_HAVE_LIBZSTD
376 /* Try to open as zstd file */
377 fp->u.zstd.dstream = ZSTD_createDStream();
378
379 fp->u.zstd.plain = fc_fopen(filename, test_mode);
380 if (fp->u.zstd.plain) {
381 size_t len = 0;
382
383 fp->u.zstd.in_buf.size = ZSTD_initDStream(fp->u.zstd.dstream);
384 fp->u.zstd.nonconst_in = fc_malloc(fp->u.zstd.in_buf.size);
385 fp->u.zstd.in_buf.src = fp->u.zstd.nonconst_in;
386
387 len = fread(fp->u.zstd.nonconst_in, 1, fp->u.zstd.in_buf.size,
388 fp->u.zstd.plain);
389 if (len > 0) {
390 if (len < fp->u.zstd.in_buf.size) {
391 fp->u.zstd.in_buf.size = len;
392 }
393 fp->u.zstd.out_buf.size = ZSTD_DStreamOutSize();
394 fp->u.zstd.out_buf.dst = fc_malloc(fp->u.zstd.out_buf.size);
395
396 fp->u.zstd.out_buf.pos = 0;
397 fp->u.zstd.in_buf.pos = 0;
398
399 fp->u.zstd.error = ZSTD_decompressStream(fp->u.zstd.dstream,
400 &fp->u.zstd.out_buf,
401 &fp->u.zstd.in_buf);
402
403 if (!ZSTD_isError(fp->u.zstd.error)) {
404 fp->method = FZ_ZSTD;
405 fp->u.zstd.outbuf_pos = 0;
406 return fp;
407 }
408
409 free(fp->u.zstd.out_buf.dst);
410 }
411
412 free(fp->u.zstd.nonconst_in);
413 fclose(fp->u.zstd.plain);
414 }
415
416 ZSTD_freeDStream(fp->u.zstd.dstream);
417#endif /* FREECIV_HAVE_LIBZSTD */
418
419#ifdef FREECIV_HAVE_LIBZ
420 method = FZ_ZLIB;
421#else
422 method = FZ_PLAIN;
423#endif
424 }
425
426 fp->method = fz_method_validate(method);
427
428 switch (fp->method) {
429#ifdef FREECIV_HAVE_LIBLZMA
430 case FZ_XZ:
431 {
433
434 /* xz files are binary files, so we should add "b" to mode! */
435 sz_strlcat(mode, "b");
436 memset(&fp->u.xz.stream, 0, sizeof(lzma_stream));
438 fp->u.xz.error = ret;
439 if (ret != LZMA_OK) {
440 free(fp);
441 return nullptr;
442 }
443 fp->u.xz.in_buf = fc_malloc(PLAIN_FILE_BUF_SIZE_XZ);
444 fp->u.xz.stream.next_in = fp->u.xz.in_buf;
445 fp->u.xz.out_buf = fc_malloc(PLAIN_FILE_BUF_SIZE_XZ);
446 fp->u.xz.stream.next_out = fp->u.xz.out_buf;
447 fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE_XZ;
448 fp->u.xz.out_index = 0;
449 fp->u.xz.total_read = 0;
450 fp->u.xz.plain = fc_fopen(filename, mode);
451 if (!fp->u.xz.plain) {
452 free(fp);
453 return nullptr;
454 }
455 }
456 return fp;
457#endif /* FREECIV_HAVE_LIBLZMA */
458#ifdef FREECIV_HAVE_LIBZSTD
459 case FZ_ZSTD:
460 {
461 /* zstd files are binary files, so we should add "b" to mode! */
462 sz_strlcat(mode, "b");
463
464 fp->u.zstd.plain = fc_fopen(filename, mode);
465 if (!fp->u.zstd.plain) {
466 free(fp);
467 return nullptr;
468 }
469
470 fp->u.zstd.cstream = ZSTD_createCStream();
471
472 /* As compress_level parameter is in range 0 - 9, and zstd takes 0 - 22,
473 * we scale it a bit */
474 ZSTD_initCStream(fp->u.zstd.cstream, compress_level * 2);
475
476 fp->u.zstd.in_buf.size = PLAIN_FILE_BUF_SIZE_ZSTD;
477 fp->u.zstd.nonconst_in = fc_malloc(fp->u.zstd.in_buf.size);
478 fp->u.zstd.in_buf.src = fp->u.zstd.nonconst_in;
479 fp->u.zstd.out_buf.size = ZSTD_CStreamOutSize();
480 fp->u.zstd.out_buf.dst = fc_malloc(fp->u.zstd.out_buf.size);
481 fp->u.zstd.in_buf.pos = 0;
482 fp->u.zstd.out_buf.pos = 0;
483 }
484 return fp;
485#endif /* FREECIV_HAVE_LIBZSTD */
486#ifdef FREECIV_HAVE_LIBBZ2
487 case FZ_BZIP2:
488 /* bz2 files are binary files, so we should add "b" to mode! */
489 sz_strlcat(mode, "b");
490 fp->u.bz2.plain = fc_fopen(filename, mode);
491 if (fp->u.bz2.plain) {
492 /* Open for read handled earlier */
493 fc_assert_ret_val('w' == mode[0], nullptr);
494 fp->u.bz2.file = BZ2_bzWriteOpen(&fp->u.bz2.error, fp->u.bz2.plain,
495 compress_level, 1, 15);
496 if (fp->u.bz2.error != BZ_OK) {
497 int tmp_err; /* See comments for similar variable
498 * near BZ2_bzReadOpen() */
499
500 BZ2_bzWriteClose(&tmp_err, fp->u.bz2.file, 0, nullptr, nullptr);
501 fp->u.bz2.file = nullptr;
502 }
503 } else {
504 fp->u.bz2.file = nullptr;
505 }
506 if (!fp->u.bz2.file) {
507 if (fp->u.bz2.plain) {
508 fclose(fp->u.bz2.plain);
509 }
510 free(fp);
511 fp = nullptr;
512 }
513 return fp;
514#endif /* FREECIV_HAVE_LIBBZ2 */
515#ifdef FREECIV_HAVE_LIBZ
516 case FZ_ZLIB:
517 /* gz files are binary files, so we should add "b" to mode! */
518 sz_strlcat(mode, "b");
519 if (mode[0] == 'w') {
520 cat_snprintf(mode, sizeof(mode), "%d", compress_level);
521 }
522 fp->u.zlib = fc_gzopen(filename, mode);
523 if (!fp->u.zlib) {
524 free(fp);
525 fp = nullptr;
526 }
527 return fp;
528#endif /* FREECIV_HAVE_LIBZ */
529 case FZ_PLAIN:
530 fp->u.plain = fc_fopen(filename, mode);
531 if (!fp->u.plain) {
532 free(fp);
533 fp = nullptr;
534 }
535 return fp;
536 }
537
538 /* Should never happen */
539 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
540 __FUNCTION__, fp->method);
541 free(fp);
542 return nullptr;
543}
544
545/************************************************************************/
549{
550 fz_FILE *fp;
551
552 if (!stream) {
553 return nullptr;
554 }
555
556 fp = fc_malloc(sizeof(*fp));
557 fp->method = FZ_PLAIN;
558 fp->memory = FALSE;
559 fp->u.plain = stream;
560 return fp;
561}
562
563/************************************************************************/
572{
573 int error = 0;
574
575 fc_assert_ret_val(fp != nullptr, 1);
576
577 if (fp->memory) {
578 if (fp->u.mem.control) {
579 FC_FREE(fp->u.mem.buffer);
580 }
581 FC_FREE(fp);
582
583 return 0;
584 }
585
586 switch (fz_method_validate(fp->method)) {
587#ifdef FREECIV_HAVE_LIBLZMA
588 case FZ_XZ:
589 if (fp->mode == 'w' && !xz_outbuffer_to_file(fp, LZMA_FINISH)) {
590 error = 1;
591 }
592 lzma_end(&fp->u.xz.stream);
593 free(fp->u.xz.in_buf);
594 free(fp->u.xz.out_buf);
595 fclose(fp->u.xz.plain);
596 free(fp);
597 return error;
598#endif /* FREECIV_HAVE_LIBLZMA */
599#ifdef FREECIV_HAVE_LIBZSTD
600 case FZ_ZSTD:
601 if (fp->mode == 'w') {
602 fp->u.zstd.error = ZSTD_endStream(fp->u.zstd.cstream, &fp->u.zstd.out_buf);
603 while (fp->u.zstd.out_buf.pos > 0) {
604 fwrite(fp->u.zstd.out_buf.dst, 1,
605 fp->u.zstd.out_buf.pos, fp->u.zstd.plain);
606 fp->u.zstd.out_buf.pos = 0;
607 fp->u.zstd.error = ZSTD_flushStream(fp->u.zstd.cstream, &fp->u.zstd.out_buf);
608 }
609 ZSTD_freeCStream(fp->u.zstd.cstream);
610 } else {
611 ZSTD_freeDStream(fp->u.zstd.dstream);
612 }
613 free(fp->u.zstd.nonconst_in);
614 free(fp->u.zstd.out_buf.dst);
615 fclose(fp->u.zstd.plain);
616 free(fp);
617 return 0;
618#endif /* FREECIV_HAVE_LIBZSTD */
619#ifdef FREECIV_HAVE_LIBBZ2
620 case FZ_BZIP2:
621 if ('w' == fp->mode) {
622 BZ2_bzWriteClose(&fp->u.bz2.error, fp->u.bz2.file, 0, nullptr, nullptr);
623 } else {
624 BZ2_bzReadClose(&fp->u.bz2.error, fp->u.bz2.file);
625 }
626 error = fp->u.bz2.error;
627 fclose(fp->u.bz2.plain);
628 free(fp);
629 return BZ_OK == error ? 0 : 1;
630#endif /* FREECIV_HAVE_LIBBZ2 */
631#ifdef FREECIV_HAVE_LIBZ
632 case FZ_ZLIB:
633 error = gzclose(fp->u.zlib);
634 free(fp);
635 return 0 > error ? error : 0; /* Only negative Z values are errors. */
636#endif /* FREECIV_HAVE_LIBZ */
637 case FZ_PLAIN:
638 error = fclose(fp->u.plain);
639 free(fp);
640 return error;
641 }
642
643 /* Should never happen */
644 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
645 __FUNCTION__, fp->method);
646 free(fp);
647 return 1;
648}
649
650/************************************************************************/
655char *fz_fgets(char *buffer, int size, fz_FILE *fp)
656{
657 fc_assert_ret_val(fp != nullptr, nullptr);
658
659 if (fp->memory) {
660 int i, j;
661
662 for (i = fp->u.mem.pos, j = 0;
663 i < fp->u.mem.size && j < size - 1 /* Space for '\0' */
664 && fp->u.mem.buffer[i] != '\n'
665 && (fp->u.mem.buffer[i] != '\r'
666 || fp->u.mem.size == i + 1
667 || fp->u.mem.buffer[i + 1] != '\n'); i++) {
668 buffer[j++] = fp->u.mem.buffer[i];
669 }
670
671 if (j < size - 2) {
672 /* Space for both newline and terminating '\0' */
673 if (i + 1 < fp->u.mem.size
674 && fp->u.mem.buffer[i] == '\r'
675 && fp->u.mem.buffer[i + 1] == '\n') {
676 i += 2;
677 buffer[j++] = '\n';
678 } else if (i < fp->u.mem.size
679 && fp->u.mem.buffer[i] == '\n') {
680 i++;
681 buffer[j++] = '\n';
682 }
683 }
684
685 if (j == 0) {
686 return nullptr;
687 }
688
689 fp->u.mem.pos = i;
690 buffer[j] = '\0';
691
692 return buffer;
693 }
694
695 switch (fz_method_validate(fp->method)) {
696#ifdef FREECIV_HAVE_LIBLZMA
697 case FZ_XZ:
698 {
699 int i, j;
700
701 for (i = 0; i < size - 1; i += j) {
702 size_t len = 0;
703 bool line_end;
704
705 for (j = 0, line_end = FALSE; fp->u.xz.out_avail > 0
706 && !line_end
707 && j < size - i - 1;
708 j++, fp->u.xz.out_avail--) {
709 buffer[i + j] = fp->u.xz.out_buf[fp->u.xz.out_index++];
710 fp->u.xz.total_read++;
711 if (buffer[i + j] == '\n') {
712 line_end = TRUE;
713 }
714 }
715
716 if (line_end || size <= j + i + 1) {
717 buffer[i + j] = '\0';
718 return buffer;
719 }
720
721 if (fp->u.xz.hack_byte_used) {
722 size_t hblen = 0;
723
724 fp->u.xz.in_buf[0] = fp->u.xz.hack_byte;
725 len = fread(fp->u.xz.in_buf + 1, 1, PLAIN_FILE_BUF_SIZE_XZ - 1,
726 fp->u.xz.plain);
727 len++;
728
729 if (len <= 1) {
730 hblen = fread(&fp->u.xz.hack_byte, 1, 1, fp->u.xz.plain);
731 }
732 if (hblen == 0) {
733 fp->u.xz.hack_byte_used = FALSE;
734 }
735 }
736 if (len == 0) {
737 if (fp->u.xz.error == LZMA_STREAM_END) {
738 if (i + j == 0) {
739 /* Plain file read complete, and there was nothing in xz buffers
740 -> end-of-file. */
741 return nullptr;
742 }
743 buffer[i + j] = '\0';
744 return buffer;
745 } else {
746 fp->u.xz.stream.next_out = fp->u.xz.out_buf;
747 fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE_XZ;
749 fp->u.xz.out_index = 0;
750 fp->u.xz.out_avail =
751 fp->u.xz.stream.total_out - fp->u.xz.total_read;
752 if (fp->u.xz.error != LZMA_OK
753 && fp->u.xz.error != LZMA_STREAM_END) {
754 return nullptr;
755 }
756 }
757 } else {
759
760 fp->u.xz.stream.next_in = fp->u.xz.in_buf;
761 fp->u.xz.stream.avail_in = len;
762 fp->u.xz.stream.next_out = fp->u.xz.out_buf;
763 fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE_XZ;
764 if (fp->u.xz.hack_byte_used) {
766 } else {
768 }
769 xz_action(fp, action);
770 fp->u.xz.out_avail =
771 fp->u.xz.stream.total_out - fp->u.xz.total_read;
772 fp->u.xz.out_index = 0;
773 if (fp->u.xz.error != LZMA_OK && fp->u.xz.error != LZMA_STREAM_END) {
774 return nullptr;
775 }
776 }
777 }
778
779 buffer[i] = '\0';
780 return buffer;
781 }
782 break;
783#endif /* FREECIV_HAVE_LIBLZMA */
784#ifdef FREECIV_HAVE_LIBZSTD
785 case FZ_ZSTD:
786 {
787 int i;
788
789 for (i = 0; i < size - 1;) {
790 int j;
791 size_t len = 0;
792
793 while (fp->u.zstd.outbuf_pos < fp->u.zstd.out_buf.pos) {
794 buffer[i] = ((char *)fp->u.zstd.out_buf.dst)[fp->u.zstd.outbuf_pos++];
795 if (buffer[i] == '\n' || i == size - 2) {
796 buffer[i + 1] = '\0';
797 return buffer;
798 }
799
800 i++;
801 }
802
803 fp->u.zstd.outbuf_pos = 0;
804
805 if (fp->u.zstd.in_buf.pos != 0) {
806 /* Move in-buffer */
807 for (j = 0; j < fp->u.zstd.in_buf.size - fp->u.zstd.in_buf.pos; j++) {
808 fp->u.zstd.nonconst_in[j] = fp->u.zstd.nonconst_in[j + fp->u.zstd.in_buf.pos];
809 }
810
811 /* Fill in-buffer from plain file */
812 len = fread(fp->u.zstd.nonconst_in + j, 1, fp->u.zstd.in_buf.size - j,
813 fp->u.zstd.plain);
814
815 if (len + j < fp->u.zstd.in_buf.size) {
816 fp->u.zstd.in_buf.size = len + j;
817 }
818 }
819
820 fp->u.zstd.out_buf.pos = 0;
821 fp->u.zstd.in_buf.pos = 0;
822
823 fp->u.zstd.error = ZSTD_decompressStream(fp->u.zstd.dstream,
824 &fp->u.zstd.out_buf,
825 &fp->u.zstd.in_buf);
826 if (ZSTD_isError(fp->u.zstd.error)) {
827 /* zstd error */
828 return nullptr;
829 }
830
831 if (fp->u.zstd.out_buf.pos == 0 && len == 0) {
832 /* Plain file fully read, and decompression outbuffer drained. */
833 if (i == 0) {
834 return nullptr;
835 }
836
837 buffer[i] = '\0';
838 return buffer;
839 }
840 }
841
842 buffer[i] = '\0';
843 return buffer;
844 }
845 break;
846#endif /* FREECIV_HAVE_LIBZSTD */
847#ifdef FREECIV_HAVE_LIBBZ2
848 case FZ_BZIP2:
849 {
850 char *retval = nullptr;
851 int i = 0;
852 int last_read;
853
854 /* See if first byte is already read and stored */
855 if (fp->u.bz2.firstbyte >= 0) {
856 buffer[0] = fp->u.bz2.firstbyte;
857 fp->u.bz2.firstbyte = -1;
858 i++;
859 } else {
860 if (!fp->u.bz2.eof) {
861 last_read = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file, buffer + i, 1);
862 i += last_read; /* 0 or 1 */
863 }
864 }
865 if (!fp->u.bz2.eof) {
866 /* Leave space for trailing zero */
867 for (; i < size - 1
868 && fp->u.bz2.error == BZ_OK && buffer[i - 1] != '\n' ;
869 i += last_read) {
870 last_read = BZ2_bzRead(&fp->u.bz2.error, fp->u.bz2.file,
871 buffer + i, 1);
872 }
873 if (fp->u.bz2.error != BZ_OK
874 && (fp->u.bz2.error != BZ_STREAM_END
875 || i == 0)) {
876 retval = nullptr;
877 } else {
878 retval = buffer;
879 }
880 if (fp->u.bz2.error == BZ_STREAM_END) {
881 /* EOF reached. Do not BZ2_bzRead() any more. */
882 fp->u.bz2.eof = TRUE;
883 }
884 }
885 buffer[i] = '\0';
886 return retval;
887 }
888#endif /* FREECIV_HAVE_LIBBZ2 */
889#ifdef FREECIV_HAVE_LIBZ
890 case FZ_ZLIB:
891 return gzgets(fp->u.zlib, buffer, size);
892#endif /* FREECIV_HAVE_LIBZ */
893 case FZ_PLAIN:
894 return fgets(buffer, size, fp->u.plain);
895 }
896
897 /* Should never happen */
898 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
899 __FUNCTION__, fp->method);
900 return nullptr;
901}
902
903#ifdef FREECIV_HAVE_LIBLZMA
904
905/************************************************************************/
910{
911 do {
912 size_t len;
913 size_t total = 0;
914
915 fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
916
917 if (fp->u.xz.error != LZMA_OK && fp->u.xz.error != LZMA_STREAM_END) {
918 return FALSE;
919 }
920
921 while (total < PLAIN_FILE_BUF_SIZE_XZ - fp->u.xz.stream.avail_out) {
922 len = fwrite(fp->u.xz.out_buf, 1,
923 PLAIN_FILE_BUF_SIZE_XZ - fp->u.xz.stream.avail_out - total,
924 fp->u.xz.plain);
925 total += len;
926 if (len == 0) {
927 return FALSE;
928 }
929 }
930 fp->u.xz.stream.avail_out = PLAIN_FILE_BUF_SIZE_XZ;
931 fp->u.xz.stream.next_out = fp->u.xz.out_buf;
932 } while (fp->u.xz.stream.avail_in > 0);
933
934 return TRUE;
935}
936
937/************************************************************************/
940static void xz_action(fz_FILE *fp, lzma_action action)
941{
942 fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
943 if (fp->u.xz.error != LZMA_MEMLIMIT_ERROR) {
944 return;
945 }
946
947 while (fp->u.xz.error == LZMA_MEMLIMIT_ERROR
948 && fp->u.xz.memlimit < XZ_DECODER_MEMLIMIT_FINAL) {
949 fp->u.xz.memlimit += XZ_DECODER_MEMLIMIT_STEP;
950 if (fp->u.xz.memlimit > XZ_DECODER_MEMLIMIT_FINAL) {
951 fp->u.xz.memlimit = XZ_DECODER_MEMLIMIT_FINAL;
952 }
953 fp->u.xz.error = lzma_memlimit_set(&fp->u.xz.stream, fp->u.xz.memlimit);
954 }
955
956 fp->u.xz.error = lzma_code(&fp->u.xz.stream, action);
957}
958#endif /* FREECIV_HAVE_LIBLZMA */
959
960/************************************************************************/
971int fz_fprintf(fz_FILE *fp, const char *format, ...)
972{
973 int num;
974 va_list ap;
975
976 fc_assert_ret_val(fp != nullptr, 0);
977 fc_assert_ret_val(!fp->memory, 0);
978
979 switch (fz_method_validate(fp->method)) {
980#ifdef FREECIV_HAVE_LIBLZMA
981 case FZ_XZ:
982 {
983 va_start(ap, format);
984 num = fc_vsnprintf((char *)fp->u.xz.in_buf, PLAIN_FILE_BUF_SIZE_XZ,
985 format, ap);
986 va_end(ap);
987
988 if (num == -1) {
989 log_error("Too much data: truncated in fz_fprintf (%u)",
992 }
993 fp->u.xz.stream.next_in = fp->u.xz.in_buf;
994 fp->u.xz.stream.avail_in = num;
995
996 if (!xz_outbuffer_to_file(fp, LZMA_RUN)) {
997 return 0;
998 } else {
999 return strlen((char *)fp->u.xz.in_buf);
1000 }
1001 }
1002 break;
1003#endif /* FREECIV_HAVE_LIBLZMA */
1004#ifdef FREECIV_HAVE_LIBZSTD
1005 case FZ_ZSTD:
1006 {
1007 va_start(ap, format);
1008 num = fc_vsnprintf((char *)fp->u.zstd.in_buf.src,
1009 PLAIN_FILE_BUF_SIZE_ZSTD, format, ap);
1010 va_end(ap);
1011
1012 if (num == -1) {
1013 log_error("Too much data: truncated in fz_fprintf (%u)",
1016 }
1017
1018 fp->u.zstd.in_buf.pos = 0;
1019 fp->u.zstd.in_buf.size = num;
1020
1021 while (fp->u.zstd.in_buf.pos < fp->u.zstd.in_buf.size) {
1022 int len;
1023
1024 fp->u.zstd.error = ZSTD_compressStream(fp->u.zstd.cstream,
1025 &fp->u.zstd.out_buf,
1026 &fp->u.zstd.in_buf);
1027 if (ZSTD_isError(fp->u.zstd.error)) {
1028 return 0;
1029 }
1030
1031 if (fp->u.zstd.out_buf.pos > 0) {
1032 len = fwrite(fp->u.zstd.out_buf.dst, 1,
1033 fp->u.zstd.out_buf.pos, fp->u.zstd.plain);
1034
1035 if (len < 0) {
1036 return 0;
1037 }
1038
1039 fp->u.zstd.out_buf.pos = 0;
1040 }
1041 }
1042
1043 return num;
1044 }
1045 break;
1046#endif /* FREECIV_HAVE_LIBZSTD */
1047#ifdef FREECIV_HAVE_LIBBZ2
1048 case FZ_BZIP2:
1049 {
1050 char buffer[65536];
1051
1052 va_start(ap, format);
1053 num = fc_vsnprintf(buffer, sizeof(buffer), format, ap);
1054 va_end(ap);
1055 if (num == -1) {
1056 log_error("Too much data: truncated in fz_fprintf (%lu)",
1057 (unsigned long) sizeof(buffer));
1058 }
1059 BZ2_bzWrite(&fp->u.bz2.error, fp->u.bz2.file, buffer, strlen(buffer));
1060 if (fp->u.bz2.error != BZ_OK) {
1061 return 0;
1062 } else {
1063 return strlen(buffer);
1064 }
1065 }
1066#endif /* FREECIV_HAVE_LIBBZ2 */
1067#ifdef FREECIV_HAVE_LIBZ
1068 case FZ_ZLIB:
1069 {
1070 char buffer[65536];
1071
1072 va_start(ap, format);
1073 num = fc_vsnprintf(buffer, sizeof(buffer), format, ap);
1074 va_end(ap);
1075 if (num == -1) {
1076 log_error("Too much data: truncated in fz_fprintf (%lu)",
1077 (unsigned long) sizeof(buffer));
1078 }
1079 return gzwrite(fp->u.zlib, buffer, (unsigned int)strlen(buffer));
1080 }
1081#endif /* FREECIV_HAVE_LIBZ */
1082 case FZ_PLAIN:
1083 va_start(ap, format);
1084 num = vfprintf(fp->u.plain, format, ap);
1085 va_end(ap);
1086 return num;
1087 }
1088
1089 /* Should never happen */
1090 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
1091 __FUNCTION__, fp->method);
1092 return 0;
1093}
1094
1095/************************************************************************/
1100{
1101 fc_assert_ret_val(fp != nullptr, 0);
1102
1103 if (fp->memory) {
1104 return 0;
1105 }
1106
1107 switch (fz_method_validate(fp->method)) {
1108#ifdef FREECIV_HAVE_LIBLZMA
1109 case FZ_XZ:
1110 if (fp->u.xz.error != LZMA_OK
1111 && fp->u.xz.error != LZMA_STREAM_END) {
1112 return 1;
1113 }
1114
1115 return 0;
1116#endif /* FREECIV_HAVE_LZMA */
1117#ifdef FREECIV_HAVE_LIBZSTD
1118 case FZ_ZSTD:
1119 if (ZSTD_isError(fp->u.zstd.error)) {
1120 return 1;
1121 }
1122 return 0;
1123#endif /* FREECIV_HAVE_LIBZSTD */
1124#ifdef FREECIV_HAVE_LIBBZ2
1125 case FZ_BZIP2:
1126 return (BZ_OK != fp->u.bz2.error
1127 && BZ_STREAM_END != fp->u.bz2.error);
1128#endif /* FREECIV_HAVE_LIBBZ2 */
1129#ifdef FREECIV_HAVE_LIBZ
1130 case FZ_ZLIB:
1131 {
1132 int error;
1133
1134 (void) gzerror(fp->u.zlib, &error); /* Ignore string result here. */
1135 return 0 > error ? error : 0; /* Only negative Z values are errors. */
1136 }
1137#endif /* FREECIV_HAVE_LIBZ */
1138 case FZ_PLAIN:
1139 return ferror(fp->u.plain);
1140 break;
1141 }
1142
1143 /* Should never happen */
1144 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
1145 __FUNCTION__, fp->method);
1146 return 0;
1147}
1148
1149/************************************************************************/
1157const char *fz_strerror(fz_FILE *fp)
1158{
1159 fc_assert_ret_val(fp != nullptr, nullptr);
1160 fc_assert_ret_val(!fp->memory, nullptr);
1161
1162 switch (fz_method_validate(fp->method)) {
1163#ifdef FREECIV_HAVE_LIBLZMA
1164 case FZ_XZ:
1165 {
1166 static char xzerror[50];
1167 char *cleartext = nullptr;
1168
1169 switch (fp->u.xz.error) {
1170 case LZMA_OK:
1171 cleartext = "OK";
1172 break;
1173 case LZMA_STREAM_END:
1174 cleartext = "Stream end";
1175 break;
1176 case LZMA_NO_CHECK:
1177 cleartext = "No integrity check";
1178 break;
1180 cleartext = "Cannot calculate the integrity check";
1181 break;
1182 case LZMA_MEM_ERROR:
1183 cleartext = "Mem error";
1184 break;
1186 cleartext = "Memory limit reached";
1187 break;
1188 case LZMA_FORMAT_ERROR:
1189 cleartext = "Unrecognized file format";
1190 break;
1191 case LZMA_OPTIONS_ERROR:
1192 cleartext = "Unsupported options";
1193 break;
1194 case LZMA_DATA_ERROR:
1195 cleartext = "Data error";
1196 break;
1197 case LZMA_BUF_ERROR:
1198 cleartext = "Progress not possible";
1199 break;
1200 default:
1201 break;
1202 }
1203
1204 if (cleartext != nullptr) {
1205 fc_snprintf(xzerror, sizeof(xzerror), "XZ: \"%s\" (%d)",
1206 cleartext, fp->u.xz.error);
1207 } else {
1208 fc_snprintf(xzerror, sizeof(xzerror), "XZ error %d",
1209 fp->u.xz.error);
1210 }
1211 return xzerror;
1212 }
1213 break;
1214#endif /* FREECIV_HAVE_LIBLZMA */
1215#ifdef FREECIV_HAVE_LIBZSTD
1216 case FZ_ZSTD:
1217 {
1218 static char zstderror[50];
1219 char *cleartext = nullptr;
1220
1221 if (ZSTD_isError(fp->u.zstd.error)) {
1222 cleartext = "error";
1223 }
1224
1225 if (cleartext != nullptr) {
1227 "ZSTD: \"%s\" (" SIZE_T_PRINTF ")",
1228 cleartext, fp->u.zstd.error);
1229 } else {
1231 "ZSTD error " SIZE_T_PRINTF,
1232 fp->u.zstd.error);
1233 }
1234 return zstderror;
1235 }
1236 break;
1237#endif /* FREECIV_HAVE_LIBZSTD */
1238#ifdef FREECIV_HAVE_LIBBZ2
1239 case FZ_BZIP2:
1240 {
1241 static char bzip2error[50];
1242 const char *cleartext = nullptr;
1243
1244 /* Rationale for translating these:
1245 * - Some of them provide usable information to user
1246 * - Messages still contain numerical error code for developers
1247 */
1248 switch (fp->u.bz2.error) {
1249 case BZ_OK:
1250 cleartext = "OK";
1251 break;
1252 case BZ_RUN_OK:
1253 cleartext = "Run ok";
1254 break;
1255 case BZ_FLUSH_OK:
1256 cleartext = "Flush ok";
1257 break;
1258 case BZ_FINISH_OK:
1259 cleartext = "Finish ok";
1260 break;
1261 case BZ_STREAM_END:
1262 cleartext = "Stream end";
1263 break;
1264 case BZ_CONFIG_ERROR:
1265 cleartext = "Config error";
1266 break;
1267 case BZ_SEQUENCE_ERROR:
1268 cleartext = "Sequence error";
1269 break;
1270 case BZ_PARAM_ERROR:
1271 cleartext = "Parameter error";
1272 break;
1273 case BZ_MEM_ERROR:
1274 cleartext = "Mem error";
1275 break;
1276 case BZ_DATA_ERROR:
1277 cleartext = "Data error";
1278 break;
1280 cleartext = "Not bzip2 file";
1281 break;
1282 case BZ_IO_ERROR:
1283 cleartext = "IO error";
1284 break;
1285 case BZ_UNEXPECTED_EOF:
1286 cleartext = "Unexpected EOF";
1287 break;
1288 case BZ_OUTBUFF_FULL:
1289 cleartext = "Output buffer full";
1290 break;
1291 default:
1292 break;
1293 }
1294
1295 if (cleartext != nullptr) {
1296 fc_snprintf(bzip2error, sizeof(bzip2error), "Bz2: \"%s\" (%d)",
1297 cleartext, fp->u.bz2.error);
1298 } else {
1299 fc_snprintf(bzip2error, sizeof(bzip2error), "Bz2 error %d",
1300 fp->u.bz2.error);
1301 }
1302 return bzip2error;
1303 }
1304#endif /* FREECIV_HAVE_LIBBZ2 */
1305#ifdef FREECIV_HAVE_LIBZ
1306 case FZ_ZLIB:
1307 {
1308 int errnum;
1309 const char *estr = gzerror(fp->u.zlib, &errnum);
1310
1311 return Z_ERRNO == errnum ? fc_strerror(fc_get_errno()) : estr;
1312 }
1313#endif /* FREECIV_HAVE_LIBZ */
1314 case FZ_PLAIN:
1315 return fc_strerror(fc_get_errno());
1316 }
1317
1318 /* Should never happen */
1319 fc_assert_msg(FALSE, "Internal error in %s() (method = %d)",
1320 __FUNCTION__, fp->method);
1321 return nullptr;
1322}
char * incite_cost
Definition comments.c:76
#define fz_method_validate(method)
Definition ioz.c:195
char * fz_fgets(char *buffer, int size, fz_FILE *fp)
Definition ioz.c:655
int fz_fprintf(fz_FILE *fp, const char *format,...)
Definition ioz.c:971
fz_FILE * fz_from_stream(FILE *stream)
Definition ioz.c:548
fz_FILE * fz_from_memory(char *buffer, int size, bool control)
Definition ioz.c:207
fz_FILE * fz_from_file(const char *filename, const char *in_mode, enum fz_method method, int compress_level)
Definition ioz.c:230
const char * fz_strerror(fz_FILE *fp)
Definition ioz.c:1157
static bool fz_method_is_valid(enum fz_method method)
Definition ioz.c:174
int fz_fclose(fz_FILE *fp)
Definition ioz.c:571
int fz_ferror(fz_FILE *fp)
Definition ioz.c:1099
fz_method
Definition ioz.h:37
@ FZ_PLAIN
Definition ioz.h:38
#define fc_assert_msg(condition, message,...)
Definition log.h:182
#define fc_assert_ret_val(condition, val)
Definition log.h:195
#define log_error(message,...)
Definition log.h:104
#define FC_FREE(ptr)
Definition mem.h:41
#define fc_malloc(sz)
Definition mem.h:34
int len
Definition packhand.c:127
size_t size
Definition specvec.h:72
union fz_FILE_s::@1 u
char mode
Definition ioz.c:151
bool memory
Definition ioz.c:152
struct mem_fzFILE mem
Definition ioz.c:154
enum fz_method method
Definition ioz.c:150
FILE * plain
Definition ioz.c:155
int size
Definition ioz.c:146
bool control
Definition ioz.c:143
char * buffer
Definition ioz.c:144
int pos
Definition ioz.c:145
int fc_snprintf(char *str, size_t n, const char *format,...)
Definition support.c:960
const char * fc_strerror(fc_errno err)
Definition support.c:609
int cat_snprintf(char *str, size_t n, const char *format,...)
Definition support.c:986
int fc_vsnprintf(char *str, size_t n, const char *format, va_list ap)
Definition support.c:886
FILE * fc_fopen(const char *filename, const char *opentype)
Definition support.c:505
fc_errno fc_get_errno(void)
Definition support.c:592
bool is_reg_file_for_access(const char *name, bool write_access)
Definition support.c:1120
#define sz_strlcpy(dest, src)
Definition support.h:195
#define TRUE
Definition support.h:46
#define FALSE
Definition support.h:47
#define sz_strlcat(dest, src)
Definition support.h:196