Ticket #1167: colladacache+archivebuild_fixes-02172012.patch

source/graphics/ColladaManager.cpp

@@ -1 @@
-/* Copyright (C) 2011 Wildfire Games.
+/* Copyright (C) 2012 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -21 +21 @@
 
 #include "graphics/ModelDef.h"
 #include "lib/fnv_hash.h"
+#include "maths/MD5.h"
+#include "ps/CacheLoader.h"
 #include "ps/CLogger.h"
 #include "ps/CStr.h"
 #include "ps/DllLoader.h"
@@ -62 +64 @@
     int (*convert_dae_to_psa)(const char* dae, Collada::OutputFn psa_writer, void* cb_data);
 
 public:
-    CColladaManagerImpl()
-        : dll("Collada")
+    CColladaManagerImpl(const PIVFS& vfs)
+        : dll("Collada"), m_VFS(vfs)
     {
     }
 
@@ -109 +111 @@
             set_logger(ColladaLog, static_cast<void*>(&skeletonPath));
 
             CVFSFile skeletonFile;
-            if (skeletonFile.Load(g_VFS, skeletonPath) != PSRETURN_OK)
+            if (skeletonFile.Load(m_VFS, skeletonPath) != PSRETURN_OK)
             {
                 LOGERROR(L"Failed to read skeleton definitions");
                 dll.Unload();
@@ -123 +125 @@
                 dll.Unload();
                 return false;
             }
-
-            // TODO: the cached PMD/PSA files should probably be invalidated when
-            // the skeleton definition file is changed, else people will get confused
-            // as to why it's not picking up their changes
         }
 
         // Set the filename for the logger to report
@@ -137 +135 @@
         CStr daeData;
         {
             CVFSFile daeFile;
-            if (daeFile.Load(g_VFS, daeFilename) != PSRETURN_OK)
+            if (daeFile.Load(m_VFS, daeFilename) != PSRETURN_OK)
                 return false;
             daeData = daeFile.GetAsString();
         }
 
         // Do the conversion into a memory buffer
+        // We need to check the result, as archive builder needs to know if the source dae
+        //  was sucessfully converted to .pmd/psa
+        int result = -1;
         WriteBuffer writeBuffer;
         switch (type)
         {
-        case CColladaManager::PMD: convert_dae_to_pmd(daeData.c_str(), ColladaOutput, &writeBuffer); break;
-        case CColladaManager::PSA: convert_dae_to_psa(daeData.c_str(), ColladaOutput, &writeBuffer); break;
+        case CColladaManager::PMD:
+            result = convert_dae_to_pmd(daeData.c_str(), ColladaOutput, &writeBuffer);
+            break;
+        case CColladaManager::PSA:
+            result = convert_dae_to_psa(daeData.c_str(), ColladaOutput, &writeBuffer);
+            break;
         }
 
         // don't create zero-length files (as happens in test_invalid_dae when
@@ -155 +160 @@
         // logic warns when asked to load such.
         if (writeBuffer.Size())
         {
-            Status ret = g_VFS->CreateFile(pmdFilename, writeBuffer.Data(), writeBuffer.Size());
+            Status ret = m_VFS->CreateFile(pmdFilename, writeBuffer.Data(), writeBuffer.Size());
             ENSURE(ret == INFO::OK);
         }
 
-        return true;
+        return (result == 0);
     }
+
+private:
+    PIVFS m_VFS;
 };
 
-CColladaManager::CColladaManager()
-: m(new CColladaManagerImpl())
+CColladaManager::CColladaManager(const PIVFS& vfs)
+: m(new CColladaManagerImpl(vfs)), m_VFS(vfs)
 {
 }
 
@@ -173 +181 @@
     delete m;
 }
 
-VfsPath CColladaManager::GetLoadableFilename(const VfsPath& pathnameNoExtension, FileType type)
+void CColladaManager::PrepareCacheKey(MD5& hash, u32& version)
 {
+    // Include skeletons.xml file info in the hash
+    VfsPath skeletonPath("art/skeletons/skeletons.xml");
+    FileInfo fileInfo;
+    if (m_VFS->GetFileInfo(skeletonPath, &fileInfo) != INFO::OK)
+    {
+        // This shouldn't occur for any sensible reasons
+        LOGERROR(L"Failed to stat '%ls' for DAE caching", skeletonPath.string().c_str());
+        // We can continue, something else will break if we try loading a skeletal model
+    }
+    else
+    {
+        u64 skeletonsModifyTime = (u64)fileInfo.MTime() & ~1; // skip lowest bit, since zip and FAT don't preserve it
+        u64 skeletonsSize = (u64)fileInfo.Size();   
+        hash.Update((const u8*)&skeletonsModifyTime, sizeof(skeletonsModifyTime));
+        hash.Update((const u8*)&skeletonsSize, sizeof(skeletonsSize));
+    }
+
+    // Add converter version to the hash
+    int converterVersion = COLLADA_CONVERTER_VERSION;
+    hash.Update((const u8*)&converterVersion, sizeof(converterVersion));
+
+    // Arbitrary version number - change this if we update the code and
+    // need to invalidate old users' caches
+    version = 1;
+}
+
+VfsPath CColladaManager::GetLoadablePath(const VfsPath& pathnameNoExtension, FileType type)
+{
     std::wstring extn;
     switch (type)
     {
@@ -186 +222 @@
     /*
 
     If there is a .dae file:
-        * Calculate a hash to identify it.
-        * Look for a cached .pmd file matching that hash.
-        * If it exists, load it. Else, convert the .dae into .pmd and load it.
-    Otherwise, if there is a (non-cache) .pmd file:
-        * Load it.
-    Else, fail.
+        * Calculate hash of skeletons.xml and converter version.
+        * Use CCacheLoader to check for archived or loose cached .pmd/psa.
+        * If it exists:
+            * Return pathname of .pmd/psa.
+        * Else:
+            * Convert the .dae into .pmd/psa.
+            * Return pathname of .pmd/psa.
+    Else, if there is a (non-cached) .psa/pmd file:
+        * Return pathname of .pmd/psa.
+    Else:
+        * Fail (return empty string).
 
-    The hash calculation ought to be fast, since normally (during development)
-    the .dae file will exist but won't have changed recently and so the cache
-    would be used. Hence, just hash the file's size, mtime, and the converter
-    version number (so updates of the converter can cause regeneration of .pmds)
-    instead of the file's actual contents.
+    Since we use CCacheLoader which automatically hashes file size and mtime,
+    and handles archived files and loose cache, when preparing the cache key
+    we add converter version number (so updates of the converter cause
+    regeneration of the .pmd/psa) and the global skeletons.xml file size and
+    mtime, as modelers frequently change the contents of skeletons.xml and get
+    perplexed if the in-game models haven't updated as expected (we don't know
+    which models were affected by the skeletons.xml change, if any, so we just
+    regenerate all of them)
 
-    TODO (maybe): The .dae -> .pmd conversion may fail (e.g. if the .dae is
+    TODO (maybe): The .dae -> .pmd/psa conversion may fail (e.g. if the .dae is
     invalid or unsupported), but it may take a long time to start the conversion
     then realise it's not going to work. That will delay the loading of the game
     every time, which is annoying, so maybe it should cache the error message
@@ -208 +252 @@
 
     */
 
-    // (TODO: the comments and variable names say "pmd" but actually they can
-    // be "psa" too.)
-
-    VfsPath dae(pathnameNoExtension.ChangeExtension(L".dae"));
-    if (! VfsFileExists(dae))
+    // Check if the .dae source exists
+    VfsPath sourcePath = pathnameNoExtension.ChangeExtension(L".dae");
+    if (m_VFS->GetFileInfo(sourcePath, 0) != INFO::OK)
     {
-        // No .dae - got to use the .pmd, assuming there is one
+        // No .dae source file - have to use the uncached .pmd/psa if it exists
+        // (cached .pmd/psa are only created by converting DAEs)
         return pathnameNoExtension.ChangeExtension(extn);
     }
 
-    // There is a .dae - see if there's an up-to-date cached copy
+    // Now we're looking for cached files
+    CCacheLoader cacheLoader(m_VFS, extn);
+    MD5 hash;
+    u32 version;
+    PrepareCacheKey(hash, version);
 
-    FileInfo fileInfo;
-    if (g_VFS->GetFileInfo(dae, &fileInfo) < 0)
+    VfsPath loadPath;
+    Status ret = cacheLoader.TryLoadingCached(sourcePath, hash, version, loadPath);
+    if (ret == INFO::OK)
     {
-        // This shouldn't occur for any sensible reasons
-        LOGERROR(L"Failed to stat DAE file '%ls'", dae.string().c_str());
-        return VfsPath();
+        // Found a cached version
+        return loadPath;
     }
+    else if (ret == INFO::SKIPPED)
+    {
+        // No cached version was found - we'll need to create it
+    }
+    else
+    {
+        ENSURE(ret < 0);
+        // This shouldn't happen, we already checked that the .dae exists
+        return L"";
+    }
 
-    // Build a struct of all the data we want to hash.
-    // (Use ints and not time_t/off_t because we don't care about overflow
-    // but do care about the fields not being 64-bit aligned)
-    // (Remove the lowest bit of mtime because some things round it to a
-    // resolution of 2 seconds)
-#pragma pack(push, 1)
-    struct { int version; int mtime; int size; } hashSource
-        = { COLLADA_CONVERTER_VERSION, (int)fileInfo.MTime() & ~1, (int)fileInfo.Size() };
-    cassert(sizeof(hashSource) == sizeof(int) * 3); // no padding, because that would be bad
-#pragma pack(pop)
+    // Cached version isn't up to date with DAE or skeletons.xml, so regenerate it
+    if (! m->Convert(sourcePath, loadPath, type))
+        return L"";
 
-    // Calculate the hash, convert to hex
-    u32 hash = fnv_hash(static_cast<void*>(&hashSource), sizeof(hashSource));
-    wchar_t hashString[9];
-    swprintf_s(hashString, ARRAY_SIZE(hashString), L"%08x", hash);
-    std::wstring extension(L"_");
-    extension += hashString;
-    extension += extn;
+    return loadPath;
+}
 
-    // realDaePath_ is "[..]/mods/whatever/art/meshes/whatever.dae"
-    OsPath realDaePath_;
-    Status ret = g_VFS->GetRealPath(dae, realDaePath_);
-    ENSURE(ret == INFO::OK);
-    wchar_t realDaeBuf[PATH_MAX];
-    wcscpy_s(realDaeBuf, ARRAY_SIZE(realDaeBuf), realDaePath_.string().c_str());
-    std::replace(realDaeBuf, realDaeBuf+ARRAY_SIZE(realDaeBuf), '\\', '/');
-    const wchar_t* realDaePath = wcsstr(realDaeBuf, L"mods/");
-
-    // cachedPmdVfsPath is "cache/mods/whatever/art/meshes/whatever_{hash}.pmd"
-    VfsPath cachedPmdVfsPath = VfsPath("cache") / realDaePath;
-    cachedPmdVfsPath = cachedPmdVfsPath.ChangeExtension(extension);
-
-    // If it's not in the cache, we'll have to create it first
-    if (! VfsFileExists(cachedPmdVfsPath))
+bool CColladaManager::GenerateCachedFile(const VfsPath& sourcePath, FileType type, VfsPath& archiveCachePath)
+{
+    std::wstring extn;
+    switch (type)
     {
-        if (! m->Convert(dae, cachedPmdVfsPath, type))
-            return L""; // failed to convert
+    case PMD: extn = L".pmd"; break;
+    case PSA: extn = L".psa"; break;
+        // no other alternatives
     }
 
-    return cachedPmdVfsPath;
+    CCacheLoader cacheLoader(m_VFS, extn);
+    MD5 hash;
+    u32 version;
+    PrepareCacheKey(hash, version);
+
+    archiveCachePath = cacheLoader.ArchiveCachePath(sourcePath);
+
+    return m->Convert(sourcePath, VfsPath("cache") / archiveCachePath, type);
 }

source/graphics/ColladaManager.h

@@ -1 @@
-/* Copyright (C) 2009 Wildfire Games.
+/* Copyright (C) 2012 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -18 +18 @@
 #ifndef INCLUDED_COLLADAMANAGER
 #define INCLUDED_COLLADAMANAGER
 
+#include "lib/file/vfs/vfs.h"
 #include "lib/file/vfs/vfs_path.h"
 
 class CStr8;
 class CColladaManagerImpl;
+class MD5;
 
 class CColladaManager
 {
 public:
     enum FileType { PMD, PSA };
 
-    CColladaManager();
+    CColladaManager(const PIVFS& vfs);
     ~CColladaManager();
 
     /**
@@ -42 +44 @@
      * @return full VFS path (including extension) of file to load; or empty
      * string if there was a problem and it could not be loaded.
      */
-    VfsPath GetLoadableFilename(const VfsPath& pathnameNoExtension, FileType type);
+    VfsPath GetLoadablePath(const VfsPath& pathnameNoExtension, FileType type);
 
+    /**
+     * Converts DAE to archive cached .pmd/psa and outputs the resulting path
+     * (used by archive builder)
+     *
+     * @param sourcePath[in] path of the .dae to load
+     * @param type[in] FileType, .pmd or .psa
+     * @param archiveCachePath[out] output path of the cached file
+     *
+     * @return true if COLLADA converter completed successfully; or false if it failed
+     */
+    bool GenerateCachedFile(const VfsPath& sourcePath, FileType type, VfsPath& archiveCachePath);
+
 private:
+    /**
+     * Creates MD5 hash key from skeletons.xml info and COLLADA converter version,
+     * used to invalidate cached .pmd/psas
+     *
+     * @param hash[out] resulting MD5 hash
+     * @param version[out] version passed to CCacheLoader, used if code change should force
+     *        cache invalidation
+     */
+    void PrepareCacheKey(MD5& hash, u32& version);
+
     CColladaManagerImpl* m;
+    PIVFS m_VFS;
 };
 
 #endif // INCLUDED_COLLADAMANAGER

source/graphics/GameView.cpp

@@ -41 +41 @@
 #include "maths/Matrix3D.h"
 #include "maths/Quaternion.h"
 #include "ps/ConfigDB.h"
+#include "ps/Filesystem.h"
 #include "ps/Game.h"
 #include "ps/Globals.h"
 #include "ps/Hotkey.h"
@@ -152 +153 @@
 public:
     CGameViewImpl(CGame* game)
         : Game(game),
-        ColladaManager(), MeshManager(ColladaManager), SkeletonAnimManager(ColladaManager),
+        ColladaManager(g_VFS), MeshManager(ColladaManager), SkeletonAnimManager(ColladaManager),
         ObjectManager(MeshManager, SkeletonAnimManager, *game->GetSimulation2()),
         LOSTexture(*game->GetSimulation2()),
         TerritoryTexture(*game->GetSimulation2()),

source/graphics/MeshManager.cpp

@@ -49 +49 @@
 
     PROFILE("load mesh");
 
-    VfsPath pmdFilename = m_ColladaManager.GetLoadableFilename(name, CColladaManager::PMD);
+    VfsPath pmdFilename = m_ColladaManager.GetLoadablePath(name, CColladaManager::PMD);
 
     if (pmdFilename.empty())
     {

source/graphics/SkeletonAnimManager.cpp

@@ -61 +61 @@
     CSkeletonAnimDef* def = NULL;
 
     // Find the file to load
-    VfsPath psaFilename = m_ColladaManager.GetLoadableFilename(name, CColladaManager::PSA);
+    VfsPath psaFilename = m_ColladaManager.GetLoadablePath(name, CColladaManager::PSA);
 
     if (psaFilename.empty())
     {

source/graphics/tests/test_MeshManager.h

@@ -1 @@
-/* Copyright (C) 2009 Wildfire Games.
+/* Copyright (C) 2012 Wildfire Games.
  * This file is part of 0 A.D.
  *
  * 0 A.D. is free software: you can redistribute it and/or modify
@@ -102 +102 @@
     void setUp()
     {
         initVfs();
-        colladaManager = new CColladaManager();
+        colladaManager = new CColladaManager(g_VFS);
         meshManager = new CMeshManager(*colladaManager);
     }
 
@@ -166 +166 @@
         copyFile(srcDAE, testDAE);
         copyFile(srcSkeletonDefs, testSkeletonDefs);
 
-        VfsPath daeName1 = colladaManager->GetLoadableFilename(testBase, CColladaManager::PMD);
-        VfsPath daeName2 = colladaManager->GetLoadableFilename(testBase, CColladaManager::PMD);
+        VfsPath daeName1 = colladaManager->GetLoadablePath(testBase, CColladaManager::PMD);
+        VfsPath daeName2 = colladaManager->GetLoadablePath(testBase, CColladaManager::PMD);
         TS_ASSERT(!daeName1.empty());
         TS_ASSERT_PATH_EQUALS(daeName1, daeName2);
         // TODO: it'd be nice to test that it really isn't doing the DAE->PMD

source/ps/ArchiveBuilder.cpp

@@ -20 +20 @@
 #include "ArchiveBuilder.h"
 
 #include "graphics/TextureManager.h"
+#include "graphics/ColladaManager.h"
 #include "lib/tex/tex_codec.h"
 #include "lib/file/archive/archive_zip.h"
 #include "lib/file/vfs/vfs_util.h"
@@ -41 +42 @@
 
     DeleteDirectory(m_TempDir/"_archivecache"); // clean up in case the last run failed
 
-    m_VFS->Mount(L"cache/", m_TempDir/"_archivecache/");
+    m_VFS->Mount(L"cache/", m_TempDir/"_archivecache"/"");
 
     m_VFS->Mount(L"", mod/"", VFS_MOUNT_MUST_EXIST | VFS_MOUNT_KEEP_DELETED);
 
@@ -76 +77 @@
 
     // Use CTextureManager instead of CTextureConverter directly,
     // so it can deal with all the loading of settings.xml files
-    CTextureManager texman(m_VFS, true, true);
+    CTextureManager textureManager(m_VFS, true, true);
 
+    CColladaManager colladaManager(m_VFS);
+
     CXeromyces xero;
 
     for (size_t i = 0; i < m_Files.size(); ++i)
@@ -99 +102 @@
         {
             VfsPath cachedPath;
             debug_printf(L"Converting texture %ls\n", realPath.string().c_str());
-            bool ok = texman.GenerateCachedTexture(path, cachedPath);
+            bool ok = textureManager.GenerateCachedTexture(path, cachedPath);
             ENSURE(ok);
 
             OsPath cachedRealPath;
@@ -113 +116 @@
             continue;
         }
 
-        // TODO: should cache DAE->PMD and DAE->PSA conversions too
+        // Cache PMD conversions of DAE models
+        if (boost::algorithm::starts_with(path.string(), L"art/meshes/") &&
+            path.Extension() == L".dae"
+        )
+        {
+            VfsPath cachedPath;
+            debug_printf(L"Converting model %ls\n", realPath.string().c_str());
+            bool ok = colladaManager.GenerateCachedFile(path, CColladaManager::PMD, cachedPath);
+            
+            // The DAE might fail to convert for whatever reason, and in that case
+            //  it can't be used in the game, so we just exclude it
+            //  (alternatively we could throw release blocking errors on useless files)
+            if (ok)
+            {
+                OsPath cachedRealPath;
+                ret = m_VFS->GetRealPath(VfsPath("cache")/cachedPath, cachedRealPath);
+                ENSURE(ret == INFO::OK);
 
+                writer->AddFile(cachedRealPath, cachedPath);
+            }
+
+            // We don't want to store the original file too (since it's a
+            // large waste of space), so skip to the next file
+            continue;
+        }
+
+        // Cache PSA conversions of DAE animations
+        if (boost::algorithm::starts_with(path.string(), L"art/animation/") &&
+            path.Extension() == L".dae"
+        )
+        {
+            VfsPath cachedPath;
+            debug_printf(L"Converting animation %ls\n", realPath.string().c_str());
+            bool ok = colladaManager.GenerateCachedFile(path, CColladaManager::PSA, cachedPath);
+            
+            // The DAE might fail to convert for whatever reason, and in that case
+            //  it can't be used in the game, so we just exclude it
+            //  (alternatively we could throw release blocking errors on useless files)
+            if (ok)
+            {
+                OsPath cachedRealPath;
+                ret = m_VFS->GetRealPath(VfsPath("cache")/cachedPath, cachedRealPath);
+                ENSURE(ret == INFO::OK);
+
+                writer->AddFile(cachedRealPath, cachedPath);
+            }
+
+            // We don't want to store the original file too (since it's a
+            // large waste of space), so skip to the next file
+            continue;
+        }
+
         debug_printf(L"Adding %ls\n", realPath.string().c_str());
         writer->AddFile(realPath, path);
 

source/tools/atlas/GameInterface/ActorViewer.cpp

@@ -37 +37 @@
 #include "graphics/UnitManager.h"
 #include "graphics/Overlay.h"
 #include "maths/MathUtil.h"
+#include "ps/Filesystem.h"
 #include "ps/Font.h"
 #include "ps/CLogger.h"
 #include "ps/GameSetup/Config.h"
@@ -61 +62 @@
     ActorViewerImpl() :
         Entity(INVALID_ENTITY),
         Terrain(),
-        ColladaManager(),
+        ColladaManager(g_VFS),
         MeshManager(ColladaManager),
         SkeletonAnimManager(ColladaManager),
         UnitManager(),