diff --git a/ape/ape.lds b/ape/ape.lds index 1d9cf917..4d8695d3 100644 --- a/ape/ape.lds +++ b/ape/ape.lds @@ -364,13 +364,18 @@ SECTIONS { PROVIDE_HIDDEN(edata = .); } :Ram +/*END: file content that's loaded by o/s */ +/*BEGIN: bss memory void */ + .zip . : { KEEP(*(SORT_BY_NAME(.zip.*))) - . = ALIGN(PAGESIZE); - HIDDEN(_efile = .); + HIDDEN(_ezip = .); } - .bss . : { +/*END: file content */ +/*BEGIN: bss memory that's addressable */ + + .bss ALIGN(__SIZEOF_POINTER__) : { KEEP(*(SORT_BY_NAME(.piro.bss.init.*))) *(.piro.bss) KEEP(*(SORT_BY_NAME(.piro.bss.sort.*))) @@ -540,8 +545,9 @@ HIDDEN(v_ape_highsectors = #define ZIPCONST(NAME, VAL) HIDDEN(NAME = DEFINED(__zip_start) ? VAL : 0); ZIPCONST(v_zip_cdoffset, __zip_start - IMAGE_BASE_VIRTUAL); ZIPCONST(v_zip_cdirsize, __zip_end - __zip_start); +ASSERT(v_zip_cdirsize % kZipCdirHdrLinkableSize == 0, "bad zip cdir"); ZIPCONST(v_zip_records, v_zip_cdirsize / kZipCdirHdrLinkableSize); -ZIPCONST(v_zip_commentsize, _efile - __zip_end - kZipCdirHdrMinSize); +ZIPCONST(v_zip_commentsize, _ezip - __zip_end - kZipCdirHdrMinSize); #if SupportsXnu() /* Generates deterministic ID. */ diff --git a/build/bootstrap/zipobj.com b/build/bootstrap/zipobj.com index 7c31177b..c9361bf7 100755 Binary files a/build/bootstrap/zipobj.com and b/build/bootstrap/zipobj.com differ diff --git a/libc/nexgen32e/zip.S b/libc/nexgen32e/zip.S index 0d377877..ba51602f 100644 --- a/libc/nexgen32e/zip.S +++ b/libc/nexgen32e/zip.S @@ -25,19 +25,17 @@ .hidden __zip_start .globl __zip_start .type __zip_start,@object - .align kZipCdirAlign __zip_start: .previous/* ... decentralized content ... */.section .zip.5,"",@progbits - .align kZipCdirAlign __zip_end: .long kZipCdirHdrMagic # magic .short 0 # disk .short 0 # starting disk - .short v_zip_records # records on disk + .short v_zip_records # number of records on disk .short v_zip_records # records .long v_zip_cdirsize # size of central directory .long RVA(__zip_start) # central directory offset diff --git a/libc/str/zipfindcentraldir.c b/libc/str/zipfindcentraldir.c index 13a76cc8..431a75ea 100644 --- a/libc/str/zipfindcentraldir.c +++ b/libc/str/zipfindcentraldir.c @@ -18,16 +18,34 @@ ╚─────────────────────────────────────────────────────────────────────────────*/ #include "libc/zip.h" -uint8_t *zipfindcentraldir(const uint8_t *map, size_t mapsize) { - const uint8_t *p, *pe; - for (p = map + mapsize - kZipCdirHdrMinSize, - pe = mapsize > 65536 + kZipCdirHdrMinSize - ? map + mapsize - 65536 - kZipCdirHdrMinSize - : map; - p >= pe; --p) { - if (ZIP_CDIR_MAGIC(p) == kZipCdirHdrMagic) { - return (/*unconst*/ uint8_t *)p; - } +/** + * Locates End Of Central Directory record in ZIP file. + * + * We search backwards for the End Of Central Directory Record magnum. + * The ZIP spec says this can be anywhere in the last 64kb. We go all + * the way since .com.dbg binaries will have lots of DWARF stuff after + * the embedded .com binary. As such, we sanity check the other fields + * too to make sure the record seems legit and it's not just a random + * occurrence of the magnum. + * + * @param p points to file memory + * @param n is byte size of file + */ +uint8_t *zipfindcentraldir(const uint8_t *p, size_t n) { + size_t i; + if (n >= kZipCdirHdrMinSize) { + i = n - kZipCdirHdrMinSize; + do { + if (ZIP_CDIR_MAGIC(p + i) == kZipCdirHdrMagic && + i + ZIP_CDIR_HDRSIZE(p + i) <= n && + ZIP_CDIR_DISK(p + i) == ZIP_CDIR_STARTINGDISK(p + i) && + ZIP_CDIR_RECORDSONDISK(p + i) == ZIP_CDIR_RECORDS(p + i) && + ZIP_CDIR_RECORDS(p + i) * kZipCdirHdrMinSize <= + ZIP_CDIR_SIZE(p + i) && + ZIP_CDIR_OFFSET(p + i) + ZIP_CDIR_SIZE(p + i) <= i) { + return (/*unconst*/ uint8_t *)(p + i); + } + } while (i--); } return NULL; } diff --git a/libc/zip.h b/libc/zip.h index c7d5ae97..00946f11 100644 --- a/libc/zip.h +++ b/libc/zip.h @@ -10,7 +10,7 @@ #define kZipAlign 2 -#define kZipCosmopolitanVersion 1 +#define kZipCosmopolitanVersion 20 #define kZipOsDos 0 #define kZipOsAmiga 1 @@ -45,7 +45,7 @@ #define kZipCdirHdrMagic 0x06054b50 /* PK♣♠ "PK\5\6" */ #define kZipCdirHdrMinSize 22 -#define kZipCdirAlign 64 /* our choice; actually 2 */ +#define kZipCdirAlign kZipAlign #define kZipCdirHdrLinkableSize \ ROUNDUP(kZipCfileHdrMinSize + PATH_MAX, kZipCdirAlign) diff --git a/libc/zipos/find.c b/libc/zipos/find.c index e3857130..fa7d7a0e 100644 --- a/libc/zipos/find.c +++ b/libc/zipos/find.c @@ -29,6 +29,7 @@ ssize_t __zipos_find(struct Zipos *zipos, const struct ZiposUri *name) { for (i = 0, cf = ZIP_CDIR_OFFSET(zipos->cdir); i < ZIP_CDIR_RECORDS(zipos->cdir); ++i, cf += ZIP_CFILE_HDRSIZE(zipos->map + cf)) { + if (ZIP_CFILE_MAGIC(zipos->map + cf) != kZipCfileHdrMagic) DebugBreak(); assert(ZIP_CFILE_MAGIC(zipos->map + cf) == kZipCfileHdrMagic); if (name->len == ZIP_CFILE_NAMESIZE(zipos->map + cf) && memcmp(name->path, ZIP_CFILE_NAME(zipos->map + cf), name->len) == 0) { diff --git a/libc/zipos/get.c b/libc/zipos/get.c index b9ae1047..985ef310 100644 --- a/libc/zipos/get.c +++ b/libc/zipos/get.c @@ -21,6 +21,7 @@ #include "libc/calls/struct/stat.h" #include "libc/limits.h" #include "libc/runtime/runtime.h" +#include "libc/str/str.h" #include "libc/sysv/consts/auxv.h" #include "libc/sysv/consts/map.h" #include "libc/sysv/consts/o.h" @@ -36,9 +37,8 @@ struct Zipos *__zipos_get(void) { static bool once; const char *exe, *dir; char path[PATH_MAX]; - size_t mapsize; - uint8_t *cdir; - void *map; + size_t size; + uint8_t *map, *base, *cdir; if (!once) { dir = nulltoempty(getenv("PWD")); /* suboptimal */ exe = (const char *)getauxval(AT_EXECFN); @@ -47,14 +47,23 @@ struct Zipos *__zipos_get(void) { exe = path; } if ((zipos.fd = open(exe, O_RDONLY)) != -1) { - if ((mapsize = getfiledescriptorsize(zipos.fd)) != SIZE_MAX && - (map = mmap(NULL, mapsize, PROT_READ, MAP_SHARED, zipos.fd, 0)) != + if ((size = getfiledescriptorsize(zipos.fd)) != SIZE_MAX && + (map = mmap(NULL, size, PROT_READ, MAP_SHARED, zipos.fd, 0)) != MAP_FAILED) { - if ((cdir = zipfindcentraldir(map, mapsize))) { - zipos.map = map; + if (endswith(exe, ".com.dbg")) { + if ((base = memmem(map, size, "MZqFpD", 6))) { + size -= base - map; + } else { + base = map; + } + } else { + base = map; + } + if ((cdir = zipfindcentraldir(base, size))) { + zipos.map = base; zipos.cdir = cdir; } else { - munmap(map, mapsize); + munmap(map, size); } } } diff --git a/test/libc/release/emulate.sh b/test/libc/release/emulate.sh index 99b7c2b2..9b3457f0 100755 --- a/test/libc/release/emulate.sh +++ b/test/libc/release/emulate.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $MODE = dbg ]; then +if [ "$MODE" = dbg ]; then exit # TODO fi diff --git a/test/libc/release/metal.sh b/test/libc/release/metal.sh index 21ab566a..e12edee5 100755 --- a/test/libc/release/metal.sh +++ b/test/libc/release/metal.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $MODE = dbg ]; then +if [ "$MODE" = dbg ]; then exit # TODO fi diff --git a/test/libc/str/undeflate_test.c b/test/libc/str/undeflate_test.c index fd7664b0..c368e009 100644 --- a/test/libc/str/undeflate_test.c +++ b/test/libc/str/undeflate_test.c @@ -93,7 +93,6 @@ TEST(undeflate, testEmbeddedCompressedZipFile_theHardWay) { ASSERT_GE(ZIP_CDIR_RECORDS(cd), 1); for (i = 0, cf = map + ZIP_CDIR_OFFSET(cd); i < ZIP_CDIR_RECORDS(cd); ++i, cf += ZIP_CFILE_HDRSIZE(cf)) { - fprintf(stderr, "%.*s\n", ZIP_CFILE_NAMESIZE(cf), ZIP_CFILE_NAME(cf)); if (strncmp("libc/testlib/hyperion.txt", ZIP_CFILE_NAME(cf), ZIP_CFILE_NAMESIZE(cf)) == 0) { lf = map + ZIP_CFILE_OFFSET(cf); diff --git a/tool/build/zipobj.c b/tool/build/zipobj.c index 76c842ec..c0fbfb0b 100644 --- a/tool/build/zipobj.c +++ b/tool/build/zipobj.c @@ -60,6 +60,10 @@ #define PUT32(P, V) \ P[0] = V & 0xff, P[1] = V >> 010 & 0xff, P[2] = V >> 020 & 0xff, \ P[3] = V >> 030 & 0xff, P += 4 +#define PUT64(P, V) \ + P[0] = V & 0xff, P[1] = V >> 010 & 0xff, P[2] = V >> 020 & 0xff, \ + P[3] = V >> 030 & 0xff, P[4] = V >> 040 & 0xff, P[5] = V >> 050 & 0xff, \ + P[6] = V >> 060 & 0xff, P[7] = V >> 070 & 0xff, P += 8 #define DOS_DATE(YEAR, MONTH_IDX1, DAY_IDX1) \ (((YEAR)-1980) << 9 | (MONTH_IDX1) << 5 | (DAY_IDX1)) @@ -149,7 +153,7 @@ static unsigned char *EmitZipLfileHdr(unsigned char *op, const void *name, uint16_t mdate, size_t compsize, size_t uncompsize) { PUT32(op, kZipLfileHdrMagic); - PUT8(op, era); + PUT8(op, kZipEra1993); PUT8(op, kZipOsDos); PUT16(op, gflags); PUT16(op, method); @@ -168,28 +172,60 @@ static void EmitZipCdirHdr(unsigned char *op, const void *name, size_t namesize, uint16_t method, uint16_t mtime, uint16_t mdate, uint16_t iattrs, uint16_t dosmode, uint16_t unixmode, size_t compsize, size_t uncompsize, - size_t commentsize) { + size_t commentsize, struct stat *st) { + uint64_t mt, at, ct; PUT32(op, kZipCfileHdrMagic); - PUT8(op, kZipCosmopolitanVersion); - PUT8(op, kZipOsUnix); - PUT8(op, era); + PUT8(op, 20); + PUT8(op, kZipOsDos); + PUT8(op, kZipEra1993); PUT8(op, kZipOsDos); PUT16(op, gflags); PUT16(op, method); PUT16(op, mtime); PUT16(op, mdate); + /* 16 */ PUT32(op, crc); PUT32(op, compsize); PUT32(op, uncompsize); PUT16(op, namesize); +#if 0 +#define CFILE_HDR_SIZE kZipCfileHdrMinSize PUT16(op, 0); /* extra size */ + /* 32 */ PUT16(op, commentsize); PUT16(op, 0); /* disk */ PUT16(op, iattrs); PUT16(op, dosmode); PUT16(op, unixmode); PUT32(op, 0); /* RELOCATE ME (kZipCfileOffsetOffset) */ + /* 46 */ memcpy(op, name, namesize); +#else +#define CFILE_HDR_SIZE (kZipCfileHdrMinSize + 36) + PUT16(op, 36); /* extra size */ + /* 32 */ + PUT16(op, commentsize); + PUT16(op, 0); /* disk */ + PUT16(op, iattrs); + PUT32(op, dosmode); + PUT32(op, 0); /* RELOCATE ME (kZipCfileOffsetOffset) */ + /* 46 */ + memcpy(op, name, namesize); + op += namesize; + PUT16(op, kZipExtraNtfs); + PUT16(op, 32); + PUT32(op, 0); + PUT16(op, 1); + PUT16(op, 24); +#define NTTIME(t) \ + (t.tv_sec + MODERNITYSECONDS) * HECTONANOSECONDS + t.tv_nsec / 100 + mt = NTTIME(st->st_mtim); + at = NTTIME(st->st_atim); + ct = NTTIME(st->st_ctim); + PUT64(op, mt); + PUT64(op, at); + PUT64(op, ct); +#endif } void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize, @@ -209,7 +245,7 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize, crc = crc32_z(0, data, uncompsize); GetDosLocalTime(st->st_mtim.tv_sec, &mtime, &mdate); gflags = IsPureAscii(name, namesize) ? 0 : kZipGflagUtf8; - commentsize = kZipCdirHdrLinkableSize - (kZipCfileHdrMinSize + namesize); + commentsize = kZipCdirHdrLinkableSize - (CFILE_HDR_SIZE + namesize); iattrs = IsPureAscii(data, st->st_size) ? kZipIattrAscii : kZipIattrBinary; dosmode = !(st->st_mode & 0200) ? kNtFileAttributeReadonly : 0; method = (st->st_size >= kMinCompressSize && ShouldCompress(name, namesize)) @@ -217,9 +253,10 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize, : kZipCompressionNone; /* emit embedded file content w/ pkzip local file header */ - elfwriter_align(elf, kZipCdirAlign, 0); - elfwriter_startsection( - elf, gc(xasprintf("%s%s", ZIP_LOCALFILE_SECTION, name)), SHT_PROGBITS, 0); + elfwriter_align(elf, 1, 0); + elfwriter_startsection(elf, + gc(xasprintf("%s%s", ZIP_LOCALFILE_SECTION, name)), + SHT_PROGBITS, SHF_ALLOC); if (method == kZipCompressionDeflate) { CHECK_EQ(Z_OK, deflateInit2(memset(&zs, 0, sizeof(zs)), Z_DEFAULT_COMPRESSION, Z_DEFLATED, -MAX_WBITS, @@ -255,12 +292,13 @@ void EmitZip(struct ElfWriter *elf, const char *name, size_t namesize, elfwriter_finishsection(elf); /* emit central directory record */ - elfwriter_align(elf, kZipCdirAlign, 0); - elfwriter_startsection( - elf, gc(xasprintf("%s%s", ZIP_DIRECTORY_SECTION, name)), SHT_PROGBITS, 0); + elfwriter_align(elf, 1, 0); + elfwriter_startsection(elf, + gc(xasprintf("%s%s", ZIP_DIRECTORY_SECTION, name)), + SHT_PROGBITS, SHF_ALLOC); EmitZipCdirHdr((cfile = elfwriter_reserve(elf, kZipCdirHdrLinkableSize)), name, namesize, crc, era, gflags, method, mtime, mdate, iattrs, - dosmode, st->st_mode, compsize, uncompsize, commentsize); + dosmode, st->st_mode, compsize, uncompsize, commentsize, st); elfwriter_appendsym(elf, gc(xasprintf("%s%s", "zip+cdir:", name)), ELF64_ST_INFO(STB_LOCAL, STT_OBJECT), STV_DEFAULT, 0, kZipCdirHdrLinkableSize);