Changeset 373 for ps

Timestamp:

06/03/04 02:17:24 (21 years ago)

Author:

janwas

Message:

some bulletproofing; completed+documented mmap

Location:

ps/trunk/source/lib/res

Files:

• file.cpp (modified) (24 diffs)
• file.h (modified) (3 diffs)
• vfs.cpp (modified) (7 diffs)
• vfs.h (modified) (3 diffs)
• zip.cpp (modified) (2 diffs)
• zip.h (modified) (2 diffs)

ps/trunk/source/lib/res/file.cpp

@@ -108 +108 @@
 //
 // can only be called once, by design (see below). rel_path is trusted.
-int file_rel_chdir(const char* argv0, const char* rel_path)
+int file_rel_chdir(const char* argv0, const char* const rel_path)
 {
     const char* msg = 0;
@@ -300 +300 @@
 
 
-int file_stat(const char* path, struct stat* s)
+int file_stat(const char* const path, struct stat* const s)
 {
     char n_path[PATH_MAX+1];
@@ -362 +362 @@
     else if(f->fd < 0)
         msg = "File fd invalid (< 0)";
+    else if((f->mapping != 0) ^ (f->map_refs != 0))
+        msg = "File mapping without refs";
 #ifndef NDEBUG
     else if(!f->fn_hash)
@@ -388 +390 @@
 
 
-int file_open(const char* p_fn, uint flags, File* f)
+int file_open(const char* const p_fn, const uint flags, File* const f)
 {
     memset(f, 0, sizeof(File));
@@ -424 +426 @@
 #endif
 
-    f->flags   = flags;
-    f->size    = size;
-    f->fn_hash = fnv_hash(n_fn);        // copy filename insteaD?
-    f->mapping = 0;
-    f->fd      = fd;
+    f->flags    = flags;
+    f->size     = size;
+    f->fn_hash  = fnv_hash(n_fn);       // copy filename instead?
+    f->mapping  = 0;
+    f->map_refs = 0;
+    f->fd       = fd;
 }
 
 invalid_f:
-    CHECK_FILE(f)
+    CHECK_FILE(f);
 
     return 0;
@@ -438 +441 @@
 
 
-int file_close(File* f)
+int file_close(File* const f)
 {
     CHECK_FILE(f);
@@ -468 +471 @@
 
 
-int ll_start_io(File* f, off_t ofs, size_t size, void* p, ll_cb* lcb)
-{
-    CHECK_FILE(f)
+int ll_start_io(File* const f, const off_t ofs, size_t size, void* const p, ll_cb* const lcb)
+{
+    CHECK_FILE(f);
 
     if(size == 0)
@@ -477 +480 @@
         return ERR_INVALID_PARAM;
     }
-    if(ofs >= f->size)
-    {
-        debug_warn("ll_start_io: ofs beyond f->size");
-        return -1;
-    }
-
-    off_t bytes_left = f->size - ofs;   // > 0
-    int op = (f->flags & FILE_WRITE)? LIO_WRITE : LIO_READ;
+
+    const int op = (f->flags & FILE_WRITE)? LIO_WRITE : LIO_READ;
 
     // cut off at EOF.
     // avoid min() due to type conversion warnings.
+    const off_t bytes_left = f->size - ofs;
+    if(bytes_left < 0)
+        return ERR_EOF;
     if((off_t)size > bytes_left)
         size = (size_t)bytes_left;
@@ -507 +507 @@
 // as a convenience, return a pointer to the transfer buffer
 // (rather than expose the ll_cb internals)
-ssize_t ll_wait_io(ll_cb* lcb, void*& p)
+ssize_t ll_wait_io(ll_cb* const lcb, void*& p)
 {
     aiocb* cb = &lcb->cb;
@@ -555 +555 @@
 
     // make sure block_num fits in 32 bits
-    size_t block_num = ofs / BLOCK_SIZE;
+    const size_t block_num = ofs / BLOCK_SIZE;
     assert(block_num <= 0xffffffff);
 
@@ -574 +574 @@
         get_mem_status();
         // TODO: calculate size
-        size_t num_blocks = 16;
+        const size_t num_blocks = 16;
 
         // evil: waste some mem (up to one block) to make sure the first block
@@ -581 +581 @@
         //
         // allocator will free the whole thing at exit.
-        void* pool = mem_alloc((num_blocks+1) * BLOCK_SIZE);
+        void* const pool = mem_alloc((num_blocks+1) * BLOCK_SIZE);
         if(!pool)
             return 0;
 
-        uintptr_t start = round_up((uintptr_t)pool + 1, BLOCK_SIZE);
+        const uintptr_t start = round_up((uintptr_t)pool + 1, BLOCK_SIZE);
             // +1 => if already block-aligned, add a whole block!
 
@@ -601 +601 @@
     }
 
-    void** entry = c.assign(id);
+    void** const entry = c.assign(id);
     if(!entry)
         return 0;
-    void* block = *entry;
+    void* const block = *entry;
 
     if(c.lock(id, true) < 0)
@@ -624 +624 @@
 static int block_retrieve(const u64 id, void*& p)
 {
-    void** entry = c.retrieve(id);
+    void** const entry = c.retrieve(id);
     if(entry)
     {
@@ -656 +656 @@
 int file_free_buf(void*& p)
 {
-    uintptr_t _p = (uintptr_t)p;
-    void* actual_p = (void*)(_p - (_p % BLOCK_SIZE));   // round down
+    const uintptr_t _p = (uintptr_t)p;
+    void* const actual_p = (void*)(_p - (_p % BLOCK_SIZE)); // round down
 
     return mem_free(actual_p);
@@ -797 +797 @@
 
 
-static int io_free(Handle hio)
+static int io_free(const Handle hio)
 {
     H_DEREF(hio, IO, io);
@@ -833 +833 @@
 struct FindBlock : public std::binary_function<Handle, u64, bool>
 {
-    bool operator()(Handle hio, u64 block_id) const
+    bool operator()(const Handle hio, const u64 block_id) const
     {
         // can't use H_DEREF - we return bool
@@ -846 +846 @@
 };
 
-static Handle io_find(u64 block_id)
+static Handle io_find(const u64 block_id)
 {
     IOList::const_iterator it;
@@ -871 +871 @@
 // transfers of more than 1 block (including padding) are allowed, but do not
 // go through the cache. don't see any case where that's necessary, though.
-Handle file_start_io(File* f, off_t user_ofs, size_t user_size, void* user_p)
+Handle file_start_io(File* const f, const off_t user_ofs, size_t user_size, void* const user_p)
 {
     int err;
 
-    CHECK_FILE(f)
+    CHECK_FILE(f);
 
     if(user_size == 0)
@@ -882 +882 @@
         return ERR_INVALID_PARAM;
     }
-    if(user_ofs >= f->size)
-    {
-        debug_warn("file_start_io: user_ofs beyond f->size");
-        return -1;
-    }
-
-    const off_t bytes_left = f->size - user_ofs;    // > 0
-    int op = (f->flags & FILE_WRITE)? LIO_WRITE : LIO_READ;
-
-    // don't read beyond EOF
-    if((off_t)user_size > bytes_left)       // avoid min() - it wants int
+
+    const int op = (f->flags & FILE_WRITE)? LIO_WRITE : LIO_READ;
+
+    // cut off at EOF.
+    // avoid min() due to type conversion warnings.
+    const off_t bytes_left = f->size - user_ofs;
+    if(bytes_left < 0)
+        return ERR_EOF;
+    if((off_t)user_size > bytes_left)
         user_size = (size_t)bytes_left;
-            // guaranteed to fit in user_size, since user_size > bytes_left
-
+        // guaranteed to fit, since size was > bytes_left
 
     u64 block_id = block_make_id(f->fn_hash, user_ofs);
@@ -925 +922 @@
 
     off_t ofs = user_ofs;
-    size_t padding = ofs % BLOCK_SIZE;
+    const size_t padding = ofs % BLOCK_SIZE;
     ofs -= (off_t)padding;
-    size_t size = round_up(padding + user_size, BLOCK_SIZE);
+    const size_t size = round_up(padding + user_size, BLOCK_SIZE);
 
 
@@ -1072 +1069 @@
 #endif
 
-    CHECK_FILE(f)
+    CHECK_FILE(f);
 
     const bool is_write = (f->flags == FILE_WRITE);
@@ -1088 +1085 @@
         if(bytes_left < 0)
             return ERR_EOF;
-        if((off_t)raw_size > bytes_left)
+        if((off_t)raw_size > bytes_left)
             raw_size = (size_t)bytes_left;
             // guaranteed to fit, since size was > bytes_left
@@ -1289 +1286 @@
 
 
-int file_map(File* f, void*& p, size_t& size)
-{
-    CHECK_FILE(f)
-
+// no significance aside from preventing uint overflow.
+static const uint MAX_MAP_REFS = 255;
+
+
+// map the entire file <f> into memory. if already currently mapped,
+// return the previous mapping (reference-counted).
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
+//
+// rationale: reference counting is required for zip_map: several
+// Zip "mappings" each reference one ZArchive's actual file mapping.
+// implement it here so that we also get refcounting for normal files.
+int file_map(File* const f, void*& p, size_t& size)
+{
+    p = 0;
+    size = 0;
+
+    CHECK_FILE(f);
+
+    // already mapped - increase refcount and return previous mapping.
+    if(f->mapping)
+    {
+        // prevent overflow; if we have this many refs, should find out why.
+        if(f->map_refs >= MAX_MAP_REFS)
+        {
+            debug_warn("file_map: too many references to mapping");
+            return -1;
+        }
+        f->map_refs++;
+        goto have_mapping;
+    }
+
+    const int prot = (f->flags & FILE_WRITE)? PROT_WRITE : PROT_READ;
+    f->mapping = mmap((void*)0, size, prot, MAP_PRIVATE, f->fd, (off_t)0);
+    if(!f->mapping)
+        return ERR_NO_MEM;
+
+    f->map_refs = 1;
+
+have_mapping:
     p = f->mapping;
     size = f->size;
-
-    // already mapped - done
-    if(p)
-        return 0;
-
-    int prot = (f->flags & FILE_WRITE)? PROT_WRITE : PROT_READ;
-
-    p = f->mapping = mmap((void*)0, (uint)size, prot, MAP_PRIVATE, f->fd, (long)0);
-    if(!p)
-    {
-        size = 0;
-        return ERR_NO_MEM;
-    }
-
     return 0;
 }
 
 
-int file_unmap(File* f)
-{
-    CHECK_FILE(f)
-
-    void* p = f->mapping;
+// decrement the reference count for the mapping belonging to file <f>.
+// fail if there are no references; remove the mapping if the count reaches 0.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
+int file_unmap(File* const f)
+{
+    CHECK_FILE(f);
+
+    void* const p = f->mapping;
+    f->mapping = 0;
+    // don't reset size - the file is still open.
+
     // not currently mapped
     if(!p)
         return -1;
-    f->mapping = 0;
-    // don't reset size - the file is still open.
 
     return munmap(p, (uint)f->size);

ps/trunk/source/lib/res/file.h

@@ -34 +34 @@
     // it is accessed by VFS and must be the same for both (union).
     // dirty, but necessary because VFile is pushing the HDATA size limit.
-    int flags;
+    uint flags;
     off_t size;
 
@@ -40 +40 @@
 
     void* mapping;
+    uint map_refs;
+
     int fd;
 };
@@ -89 +91 @@
 extern int file_close(File* f);
 
+
+//
+// memory mapping
+//
+
+// map the entire file <f> into memory. if already currently mapped,
+// return the previous mapping (reference-counted).
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
+//
+// rationale: reference counting is required for zip_map: several
+// Zip "mappings" each reference one ZArchive's actual file mapping.
+// implement it here so that we also get refcounting for normal files.
 extern int file_map(File* f, void*& p, size_t& size);
+
+// decrement the reference count for the mapping belonging to file <f>.
+// fail if there are no references; remove the mapping if the count reaches 0.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
 extern int file_unmap(File* f);
+
+
+//
+// async IO
+//
 
 extern Handle file_start_io(File* f, off_t ofs, size_t size, void* buf);

ps/trunk/source/lib/res/vfs.cpp

@@ -963 +963 @@
 
 
-static int& vf_flags(VFile* vf)
+static uint& vf_flags(VFile* vf)
 {
     assert(offsetof(struct File, flags) == offsetof(struct ZFile, flags));
@@ -973 +973 @@
 static void VFile_init(VFile* vf, va_list args)
 {
-    int flags = va_arg(args, int);
+    uint flags = va_arg(args, int);
     vf_flags(vf) = flags;
 }
@@ -980 +980 @@
 static void VFile_dtor(VFile* vf)
 {
-    int& flags = vf_flags(vf);
+    uint& flags = vf_flags(vf);
 
     if(flags & VF_OPEN)
@@ -999 +999 @@
 static int VFile_reload(VFile* vf, const char* path)
 {
-    int& flags = vf_flags(vf);
+    uint& flags = vf_flags(vf);
 
     // we're done if file is already open. need to check this because reload order
@@ -1133 +1133 @@
 
 
-int vfs_store(const char* fn, void* p, size_t size)
+int vfs_store(const char* const fn, void* p, const size_t size)
 {
     Handle hf = vfs_open(fn, VFS_WRITE);
@@ -1139 +1139 @@
         return (int)hf; // error code
     H_DEREF(hf, VFile, vf);
-    int ret = vfs_io(hf, 0, size, p);
+    const int ret = vfs_io(hf, 0, size, p);
     vfs_close(hf);
     return ret;
@@ -1145 +1145 @@
 
 
-
-
-int vfs_map(const Handle hf, uint flags, void*& p, size_t& size)
-{
-    H_DEREF(hf, VFile, vf);
-
-    if(vf_flags(vf) & VF_ZIP)
-        CHECK_ERR(zip_map(&vf->zf, p, size));
-    else
-        CHECK_ERR(file_map(&vf->f, p, size));
-    return 0;
-}
-
-
-int vfs_unmap(Handle hf)
-{
+//
+// memory mapping
+//
+
+// map the entire file <hf> into memory. if already currently mapped,
+// return the previous mapping (reference-counted).
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
+int vfs_map(const Handle hf, const uint flags, void*& p, size_t& size)
+{
+    UNUSED(flags);
+
+    p = 0;
+    size = 0;
+        // need to zero these here in case H_DEREF fails
+
     H_DEREF(hf, VFile, vf);
     if(vf_flags(vf) & VF_ZIP)
-        CHECK_ERR(zip_unmap(&vf->zf));
+        return zip_map(&vf->zf, p, size);
     else
-        CHECK_ERR(file_unmap(&vf->f));
-    return 0;
-}
+        return file_map(&vf->f, p, size);
+}
+
+
+// decrement the reference count for the mapping belonging to file <f>.
+// fail if there are no references; remove the mapping if the count reaches 0.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
+int vfs_unmap(const Handle hf)
+{
+    H_DEREF(hf, VFile, vf);
+    if(vf_flags(vf) & VF_ZIP)
+        return zip_unmap(&vf->zf);
+    else
+        return file_unmap(&vf->f);
+}

ps/trunk/source/lib/res/vfs.h

@@ -42 +42 @@
 extern int vfs_close(Handle& h);
 
+
+//
+// memory mapping
+//
+
+// map the entire file <hf> into memory. if already currently mapped,
+// return the previous mapping (reference-counted).
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
 extern int vfs_map(Handle hf, uint flags, void*& p, size_t& size);
+
+// decrement the reference count for the mapping belonging to file <f>.
+// fail if there are no references; remove the mapping if the count reaches 0.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
 extern int vfs_unmap(Handle hf);
 
+
+//
+// directory entry enumeration
+//
 
 struct vfsDirEnt
@@ -55 +78 @@
 };
 
-
 extern Handle vfs_open_dir(const char* path);
 extern int vfs_close_dir(Handle& hd);
 extern int vfs_next_dirent(Handle hd, vfsDirEnt* ent, const char* filter);
-
-
-
-
-extern int vfs_rebuild();
-
-
 
 
@@ -90 +105 @@
 
 
+
+extern int vfs_rebuild();
+
+
 #endif  // #ifndef __VFS_H__

ps/trunk/source/lib/res/zip.cpp

@@ -731 +731 @@
 
 
+enum ZFileFlags
+{
+    // the ZFile has been successfully zip_map-ped.
+    // we store this so that the archive mapping refcount remains balanced.
+    ZF_HAS_MAPPING = 0x4000
+};
+
 // marker for ZFile struct, to make sure it's valid
+#ifdef PARANOIA
 static const u32 ZFILE_MAGIC = FOURCC('Z','F','I','L');
+#endif
 
 
@@ -947 +956 @@
 
 
+// map the entire file <zf> into memory. mapping compressed files
+// isn't allowed, since the compression algorithm is unspecified.
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its file is closed.
+// however, map/unmap calls should still be paired so that the mapping
+// may be removed when no longer needed.
 int zip_map(ZFile* const zf, void*& p, size_t& size)
 {
+    p = 0;
+    size = 0;
+
     CHECK_ZFILE(zf)
 
-    // doesn't really make sense to map compressed files, so disallow it.
+    // mapping compressed files doesn't make sense because the
+    // compression algorithm is unspecified - disallow it.
     if(is_compressed(zf))
     {
-        debug_warn("mapping a compressed file from archive. why?");
+        debug_warn("zip_map: file is compressed");
         return -1;
     }
 
     H_DEREF(zf->ha, ZArchive, za)
-    // increase refs
-
-    return file_map(&za->f, p, size);
-}
-
-
+    CHECK_ERR(file_map(&za->f, p, size));
+
+    zf->flags |= ZF_HAS_MAPPING;
+    return 0;
+}
+
+
+// remove the mapping of file <zf>; fail if not mapped.
+//
+// the mapping will be removed (if still open) when its archive is closed.
+// however, map/unmap calls should be paired so that the archive mapping
+// may be removed when no longer needed.
 int zip_unmap(ZFile* const zf)
 {
     CHECK_ZFILE(zf)
+
+    // make sure archive mapping refcount remains balanced:
+    // don't allow multiple unmaps.
+    if(!(zf->flags & ZF_HAS_MAPPING))
+        return -1;
+    zf->flags &= ~ZF_HAS_MAPPING;
+
     H_DEREF(zf->ha, ZArchive, za)
-    // decrement refs
-    // unmap archive if 0
-    return 0;
-}
+    return file_unmap(&za->f);
+}

ps/trunk/source/lib/res/zip.h

@@ -74 +74 @@
     // it is accessed by VFS and must be the same for both (union).
     // dirty, but necessary because VFile is pushing the HDATA size limit.
-    int flags;
+    uint flags;
     size_t ucsize;
         // size of logical file
@@ -96 +96 @@
 
 
+//
+// memory mapping
+//
+
+// map the entire file <zf> into memory. mapping compressed files
+// isn't allowed, since the compression algorithm is unspecified.
+// output parameters are zeroed on failure.
+//
+// the mapping will be removed (if still open) when its archive is closed.
+// however, map/unmap calls should still be paired so that the archive mapping
+// may be removed when no longer needed.
 extern int zip_map(ZFile* zf, void*& p, size_t& size);
+
+// remove the mapping of file <zf>; fail if not mapped.
+//
+// the mapping will be removed (if still open) when its archive is closed.
+// however, map/unmap calls should be paired so that the archive mapping
+// may be removed when no longer needed.
 extern int zip_unmap(ZFile* zf);
 
-// read from file <zf>, starting at offset <ofs> in the compressed data
+// read from file <zf>, starting at offset <ofs> in the compressed data.
 extern ssize_t zip_read(ZFile* zf, off_t ofs, size_t size, void*& p);