From b882d8c240a1667a820adb703d3f7c84dfcd61c2 Mon Sep 17 00:00:00 2001 From: Andrei Karas Date: Thu, 23 Feb 2017 23:02:52 +0300 Subject: Add support for parsing zip archives. Also add unit tests for test it. --- src/fs/virtfs_unittest.cc | 8 +-- src/fs/virtfsdir_unittest.cc | 8 +-- src/fs/zip.cpp | 161 +++++++++++++++++++++++++++++++++++++++++++ src/fs/zip.h | 37 ++++++++++ src/fs/zip_unittest.cc | 147 +++++++++++++++++++++++++++++++++++++++ src/fs/ziplocalheader.h | 46 +++++++++++++ 6 files changed, 399 insertions(+), 8 deletions(-) create mode 100644 src/fs/zip.cpp create mode 100644 src/fs/zip.h create mode 100644 src/fs/zip_unittest.cc create mode 100644 src/fs/ziplocalheader.h (limited to 'src/fs') diff --git a/src/fs/virtfs_unittest.cc b/src/fs/virtfs_unittest.cc index b26381f08..a79f01aee 100644 --- a/src/fs/virtfs_unittest.cc +++ b/src/fs/virtfs_unittest.cc @@ -115,8 +115,8 @@ TEST_CASE("VirtFs enumerateFiles1") VirtList *list = nullptr; - const int cnt1 = VirtFs::exists("test/test2.txt") ? 24 : 23; - const int cnt2 = 24; + const int cnt1 = VirtFs::exists("test/test2.txt") ? 27 : 26; + const int cnt2 = 27; VirtFs::permitLinks(false); list = VirtFs::enumerateFiles("test"); @@ -363,8 +363,8 @@ TEST_CASE("VirtFs permitLinks") VirtFs::addDirToSearchPath("data", Append_false); VirtFs::addDirToSearchPath("../data", Append_false); - const int cnt1 = VirtFs::exists("test/test2.txt") ? 22 : 21; - const int cnt2 = 22; + const int cnt1 = VirtFs::exists("test/test2.txt") ? 25 : 24; + const int cnt2 = 25; StringVect list; VirtFs::permitLinks(false); diff --git a/src/fs/virtfsdir_unittest.cc b/src/fs/virtfsdir_unittest.cc index 91c573f61..df596712b 100644 --- a/src/fs/virtfsdir_unittest.cc +++ b/src/fs/virtfsdir_unittest.cc @@ -387,8 +387,8 @@ TEST_CASE("VirtFsDir enumerateFiles1") VirtList *list = nullptr; - const int cnt1 = VirtFsDir::exists("test/test2.txt") ? 24 : 23; - const int cnt2 = 24; + const int cnt1 = VirtFsDir::exists("test/test2.txt") ? 27 : 26; + const int cnt2 = 27; VirtFsDir::permitLinks(false); list = VirtFsDir::enumerateFiles("test"); @@ -616,8 +616,8 @@ TEST_CASE("VirtFsDir permitLinks") Append_false, SkipError_false); - const int cnt1 = VirtFsDir::exists("test/test2.txt") ? 22 : 21; - const int cnt2 = 22; + const int cnt1 = VirtFsDir::exists("test/test2.txt") ? 25 : 24; + const int cnt2 = 25; StringVect list; VirtFsDir::permitLinks(false); diff --git a/src/fs/zip.cpp b/src/fs/zip.cpp new file mode 100644 index 000000000..2164a4303 --- /dev/null +++ b/src/fs/zip.cpp @@ -0,0 +1,161 @@ +/* + * The ManaPlus Client + * Copyright (C) 2017 The ManaPlus Developers + * + * This file is part of The ManaPlus Client. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "fs/zip.h" + +#include "fs/ziplocalheader.h" + +#include "utils/checkutils.h" +#include "utils/stringutils.h" + +#include +#include + +#include "debug.h" + +#define readVal(val, sz, msg) \ + cnt = fread(static_cast(val), 1, sz, arcFile); \ + if (cnt != sz) \ + { \ + reportAlways("Error reading " msg " in file %s", \ + archiveName.c_str()); \ + delete header; \ + delete [] buf; \ + fclose(arcFile); \ + return false; \ + } \ + +namespace Zip +{ + bool readArchiveInfo(const std::string &restrict archiveName, + std::vector &restrict headers) + { + FILE *restrict const arcFile = fopen(archiveName.c_str(), + "r"); + if (arcFile == nullptr) + { + reportAlways("Can't open zip file %s", + archiveName.c_str()); + return false; + } + size_t cnt = 0U; + uint8_t *const buf = new uint8_t[65535 + 10]; + uint16_t val16 = 0U; + ZipLocalHeader *header = nullptr; + + logger->log("Read archive: %s", archiveName.c_str()); + // format source https://en.wikipedia.org/wiki/Zip_%28file_format%29 + while (feof(arcFile) == 0) + { + // file header pointer on 0 + // read file header signature + readVal(buf, 4, "zip file header"); // + 4 + // pointer on 4 + + if (buf[0] == 0x50 && + buf[1] == 0x4B && + buf[2] == 0x03 && + buf[3] == 0x04) + { // local file header + header = new ZipLocalHeader; + header->archiveName = archiveName; + // skip useless fields + fseek(arcFile, 14, SEEK_CUR); // + 14 + // file header pointer on 18 + readVal(&header->compressSize, 4, + "zip compressed size") // + 4 + // file header pointer on 22 + // +++ need add endian specific decoding for val32 + readVal(&header->uncompressSize, 4, + "zip uncompressed size") // + 4 + // file header pointer on 26 + // +++ need add endian specific decoding for val32 + readVal(&val16, 2, "file name length") // + 2 + // file header pointer on 28 + const uint32_t fileNameLen = CAST_U32(val16); + if (fileNameLen > 1000) + { + reportAlways("Error too long file name in file %s", + archiveName.c_str()); + delete header; + delete [] buf; + fclose(arcFile); + return false; + } + readVal(&val16, 2, "extra field length") // + 2 + // file header pointer on 30 + const uint32_t extraFieldLen = CAST_U32(val16); + readVal(buf, fileNameLen, "file name"); + // file header pointer on 30 + fileNameLen + buf[fileNameLen] = 0; + header->fileName = std::string( + reinterpret_cast(buf)); + header->dataOffset = ftell(arcFile) + extraFieldLen; + fseek(arcFile, extraFieldLen + header->compressSize, SEEK_CUR); + // pointer on 30 + fileNameLen + extraFieldLen + compressSize + if (findLast(header->fileName, "/") == false) + { + headers.push_back(header); + logger->log(" file name: %s", + header->fileName.c_str()); + logger->log(" compressed size: %u", header->compressSize); + logger->log(" uncompressed size: %u", header->uncompressSize); + } + } + else if (buf[0] == 0x50 && + buf[1] == 0x4B && + buf[2] == 0x01 && + buf[3] == 0x02) + { // central directory file header + // !!! This is quick way for read zip archives. !!! + // !!! It ignore modified files in archive. !!! + // ignoring central directory entries because all data already read. + break; + } + else if (buf[0] == 0x50 && + buf[1] == 0x4B && + buf[2] == 0x05 && + buf[3] == 0x06) + { // end of central directory + // !!! This is quick way for read zip archives. !!! + // !!! It ignore modified files in archive. !!! + // ignoring central directory entries because all data already read. + break; + } + else + { + reportAlways("Error in header signature (0x%02x%02x%02x%02x)" + " in file %s", + buf[0], + buf[1], + buf[2], + buf[3], + archiveName.c_str()); + delete [] buf; + fclose(arcFile); + return false; + } + } + delete [] buf; + fclose(arcFile); + return true; + } + +} // namespace Zip diff --git a/src/fs/zip.h b/src/fs/zip.h new file mode 100644 index 000000000..145be5c88 --- /dev/null +++ b/src/fs/zip.h @@ -0,0 +1,37 @@ +/* + * The ManaPlus Client + * Copyright (C) 2017 The ManaPlus Developers + * + * This file is part of The ManaPlus Client. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef UTILS_ZIP_H +#define UTILS_ZIP_H + +#include "localconsts.h" + +#include +#include + +struct ZipLocalHeader; + +namespace Zip +{ + bool readArchiveInfo(const std::string &restrict archiveName, + std::vector &restrict headers); +} // namespace Zip + +#endif // UTILS_ZIP_H diff --git a/src/fs/zip_unittest.cc b/src/fs/zip_unittest.cc new file mode 100644 index 000000000..3f0c49cdf --- /dev/null +++ b/src/fs/zip_unittest.cc @@ -0,0 +1,147 @@ +/* + * The ManaPlus Client + * Copyright (C) 2017 The ManaPlus Developers + * + * This file is part of The ManaPlus Client. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "catch.hpp" + +#include "logger.h" + +#include "fs/files.h" +#include "fs/zip.h" +#include "fs/ziplocalheader.h" + +#include "utils/delete2.h" +#include "utils/dtor.h" + +#include "debug.h" + +TEST_CASE("Zip readArchiveInfo") +{ + logger = new Logger(); + std::string name("data/test/test.zip"); + std::string prefix; + std::vector headers; + if (Files::existsLocal(name) == false) + prefix = "../"; + + SECTION("test 1") + { + name = prefix + "data/test/test.zip"; + + REQUIRE(Zip::readArchiveInfo(name, headers)); + REQUIRE(headers.size() == 2); + REQUIRE(headers[0]->archiveName == name); + REQUIRE(headers[0]->fileName == "dir/hide.png"); + REQUIRE(headers[0]->compressSize == 365); + REQUIRE(headers[0]->uncompressSize == 368); + REQUIRE(headers[1]->archiveName == name); + REQUIRE(headers[1]->fileName == "dir/brimmedhat.png"); + REQUIRE(headers[1]->compressSize == 1959); + REQUIRE(headers[1]->uncompressSize == 1959); + } + + SECTION("test 2") + { + name = prefix + "data/test/test2.zip"; + + REQUIRE(Zip::readArchiveInfo(name, headers)); + REQUIRE(headers.size() == 11); + REQUIRE(headers[0]->archiveName == name); + REQUIRE(headers[0]->fileName == "test.txt"); + REQUIRE(headers[0]->compressSize == 17); + REQUIRE(headers[0]->uncompressSize == 23); + + REQUIRE(headers[1]->archiveName == name); + REQUIRE(headers[1]->fileName == "dir2/hide.png"); + REQUIRE(headers[1]->compressSize == 365); + REQUIRE(headers[1]->uncompressSize == 368); + + REQUIRE(headers[2]->archiveName == name); + REQUIRE(headers[2]->fileName == "dir2/test.txt"); + REQUIRE(headers[2]->compressSize == 17); + REQUIRE(headers[2]->uncompressSize == 23); + + REQUIRE(headers[3]->archiveName == name); + REQUIRE(headers[3]->fileName == "dir2/paths.xml"); + REQUIRE(headers[3]->compressSize == 154); + REQUIRE(headers[3]->uncompressSize == 185); + + REQUIRE(headers[4]->archiveName == name); + REQUIRE(headers[4]->fileName == "dir2/units.xml"); + REQUIRE(headers[4]->compressSize == 202); + REQUIRE(headers[4]->uncompressSize == 306); + + REQUIRE(headers[5]->archiveName == name); + REQUIRE(headers[5]->fileName == "dir/hide.png"); + REQUIRE(headers[5]->compressSize == 365); + REQUIRE(headers[5]->uncompressSize == 368); + + REQUIRE(headers[6]->archiveName == name); + REQUIRE(headers[6]->fileName == "dir/1/test.txt"); + REQUIRE(headers[6]->compressSize == 17); + REQUIRE(headers[6]->uncompressSize == 23); + + REQUIRE(headers[7]->archiveName == name); + REQUIRE(headers[7]->fileName == "dir/1/file1.txt"); + REQUIRE(headers[7]->compressSize == 17); + REQUIRE(headers[7]->uncompressSize == 23); + + REQUIRE(headers[8]->archiveName == name); + REQUIRE(headers[8]->fileName == "dir/gpl/palette.gpl"); + REQUIRE(headers[8]->compressSize == 128); + REQUIRE(headers[8]->uncompressSize == 213); + + REQUIRE(headers[9]->archiveName == name); + REQUIRE(headers[9]->fileName == "dir/dye.png"); + REQUIRE(headers[9]->compressSize == 794); + REQUIRE(headers[9]->uncompressSize == 794); + + REQUIRE(headers[10]->archiveName == name); + REQUIRE(headers[10]->fileName == "units.xml"); + REQUIRE(headers[10]->compressSize == 202); + REQUIRE(headers[10]->uncompressSize == 306); + } + + SECTION("test 3") + { + name = prefix + "data/test/test3.zip"; + + REQUIRE(Zip::readArchiveInfo(name, headers)); + REQUIRE(headers.size() == 2); + REQUIRE(headers[0]->archiveName == name); + REQUIRE(headers[0]->fileName == "test.txt"); + REQUIRE(headers[0]->compressSize == 17); + REQUIRE(headers[0]->uncompressSize == 23); + REQUIRE(headers[1]->archiveName == name); + REQUIRE(headers[1]->fileName == "units.xml"); + REQUIRE(headers[1]->compressSize == 202); + REQUIRE(headers[1]->uncompressSize == 306); + } + + SECTION("test 4") + { + name = prefix + "data/test/test4.zip"; + + REQUIRE(Zip::readArchiveInfo(name, headers)); + REQUIRE(headers.size() == 0); + } + + delete_all(headers); + delete2(logger); +} diff --git a/src/fs/ziplocalheader.h b/src/fs/ziplocalheader.h new file mode 100644 index 000000000..1b00c3b54 --- /dev/null +++ b/src/fs/ziplocalheader.h @@ -0,0 +1,46 @@ +/* + * The ManaPlus Client + * Copyright (C) 2017 The ManaPlus Developers + * + * This file is part of The ManaPlus Client. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef UTILS_ZIPLOCALHEADER_H +#define UTILS_ZIPLOCALHEADER_H + +#include "localconsts.h" + +#include + +struct ZipLocalHeader +{ + ZipLocalHeader() : + archiveName(), + fileName(), + dataOffset(0U), + compressSize(0U), + uncompressSize(0U) + { + } + + std::string archiveName; + std::string fileName; + uint32_t dataOffset; + uint32_t compressSize; + uint32_t uncompressSize; +}; + +#endif // UTILS_ZIPLOCALHEADER_H -- cgit v1.2.3-60-g2f50