Ticket #3433: 3433_replay_cache_v1.7.patch

binaries/data/mods/public/gui/replaymenu/replay_actions.js

@@ -127 +127 @@
     });
 }
 
+function reloadCache()
+{
+    let selected = Engine.GetGUIObjectByName("replaySelection").selected;
+    loadReplays(selected > 0 ? createReplaySelectionData(g_ReplaysFiltered[selected].directory) : "", true);
+}
+
 /**
  * Callback.
  */

binaries/data/mods/public/gui/replaymenu/replay_menu.js

@@ -59 +59 @@
         return;
     }
 
-    loadReplays(data && data.replaySelectionData);
+    loadReplays(data && data.replaySelectionData, false);
 
     if (!g_Replays)
     {
@@ -76 +76 @@
  * Check timestamp and compatibility and extract g_Playernames, g_MapNames, g_VictoryConditions.
  * Restore selected filters and item.
  */
-function loadReplays(replaySelectionData)
+function loadReplays(replaySelectionData, reload)
 {
-    g_Replays = Engine.GetReplays();
+    g_Replays = Engine.GetReplays(reload);
 
     if (!g_Replays)
         return;

binaries/data/mods/public/gui/replaymenu/replay_menu.xml

@@ -250 +250 @@
                 <translatableAttribute id="caption">Delete</translatableAttribute>
                 <action on="Press">deleteReplayButtonPressed();</action>
             </object>
+            
+            <!-- Reload Cache Button -->
+            <object type="button" style="StoneButton" size="40%+25 0 57%+25 100%">
+                <translatableAttribute id="caption">Reload Cache</translatableAttribute>
+                <action on="Press">reloadCache();</action>
+            </object>
 
             <!-- Summary Button -->
             <object name="summaryButton" type="button" style="StoneButton" size="65%-50 0 82%-50 100%">

binaries/data/mods/public/gui/session/session.js

@@ -559 +559 @@
 
     Engine.EndGame();
 
+    // After the replay file was closed in EndGame
+    // Done here to keep EndGame small
+    if (!g_IsReplay)
+        Engine.AddReplayToCache(replayDirectory);
+
     if (g_IsController && Engine.HasXmppClient())
         Engine.SendUnregisterGame();
 

source/ps/scripting/JSInterface_VisualReplay.cpp

@@ -33 +33 @@
     return VisualReplay::DeleteReplay(replayFile);
 }
 
-JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate)
+JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool reload)
 {
-    return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface));
+    return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface), reload);
 }
 
 JS::Value JSI_VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
@@ -53 +53 @@
     return VisualReplay::GetReplayMetadata(pCxPrivate, directoryName);
 }
 
+void JSI_VisualReplay::AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
+{
+    VisualReplay::AddReplayToCache(*(pCxPrivate->pScriptInterface), directoryName);
+}
+
 void JSI_VisualReplay::RegisterScriptFunctions(ScriptInterface& scriptInterface)
 {
-    scriptInterface.RegisterFunction<JS::Value, &GetReplays>("GetReplays");
+    scriptInterface.RegisterFunction<JS::Value, bool, &GetReplays>("GetReplays");
     scriptInterface.RegisterFunction<bool, CStrW, &DeleteReplay>("DeleteReplay");
     scriptInterface.RegisterFunction<void, CStrW, &StartVisualReplay>("StartVisualReplay");
     scriptInterface.RegisterFunction<JS::Value, CStrW, &GetReplayAttributes>("GetReplayAttributes");
     scriptInterface.RegisterFunction<JS::Value, CStrW, &GetReplayMetadata>("GetReplayMetadata");
     scriptInterface.RegisterFunction<bool, CStrW, &HasReplayMetadata>("HasReplayMetadata");
+    scriptInterface.RegisterFunction<void, CStrW, &AddReplayToCache>("AddReplayToCache");
 }

source/ps/scripting/JSInterface_VisualReplay.h

@@ -25 +25 @@
 {
     void StartVisualReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directory);
     bool DeleteReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& replayFile);
-    JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate);
+    JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool reload);
     JS::Value GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
     bool HasReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
     JS::Value GetReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
+    void AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
     void RegisterScriptFunctions(ScriptInterface& scriptInterface);
 }
 

source/ps/VisualReplay.cpp

@@ -62 +62 @@
 }
 
 /**
- * Load all replays found in the directory.
- *
- * Since files are spread across the harddisk,
- * loading hundreds of them can consume a lot of time.
- */
-JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface)
+* Load the replay cache and check if there are new/deleted ones
+* If so, update their data.
+*/
+JS::HandleObject VisualReplay::ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFileSize)
 {
-    TIMER(L"GetReplays");
+    TIMER(L"ReloadReplayCache");
     JSContext* cx = scriptInterface.GetContext();
     JSAutoRequest rq(cx);
 
-    u32 i = 0;
+    // Maps the filename onto the index and size
+    std::map<CStr, std::pair<u32, u32>> fileList;
+
+    const OsPath cacheFileName = GetDirectoryName() / L"replayCache.json";
+    const OsPath tempCacheFileName = GetDirectoryName() / L"replayCache_temp.json";
+    JS::RootedObject cachedReplaysObject(cx);
+
+    if (FileExists(cacheFileName))
+    {
+        TIMER(L"ReloadReplayCache: openCacheFile");
+        // Open cache file
+        std::istream* cacheStream = new std::ifstream(cacheFileName.string8().c_str());
+
+        // Read file into chacheStr
+        CStr cacheStr((std::istreambuf_iterator<char>(*cacheStream)), std::istreambuf_iterator<char>());
+
+        // Create empty JS object and parse the context of the cache into it
+        JS::RootedValue cachedReplays(cx);
+        scriptInterface.ParseJSON(cacheStr, &cachedReplays);
+        SAFE_DELETE(cacheStream);
+
+        cachedReplaysObject = &cachedReplays.toObject();
+
+        // Create list of files included in the cache
+        u32 cacheLength = 0;
+        JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength);
+        for (u32 j = 0; j < cacheLength; ++j)
+        {
+            JS::RootedValue replay(cx);
+            JS_GetElement(cx, cachedReplaysObject, j, &replay);
+
+            JS::RootedValue file(cx);
+            CStr fileName;
+            u32 fileSize;
+            scriptInterface.GetProperty(replay, "directory", fileName);
+            scriptInterface.GetProperty(replay, "fileSize", fileSize);
+
+            fileList.emplace(fileName, std::make_pair(j, fileSize));
+        }
+    }
+
+    JS::RootedObject replays(cx, JS_NewArrayObject(cx, 0));
     DirectoryNames directories;
-    JS::RootedObject replays(cx, JS_NewArrayObject(cx, 0));
 
-    if (GetDirectoryEntries(GetDirectoryName(), NULL, &directories) == INFO::OK)
-        for (OsPath& directory : directories)
+    if (GetDirectoryEntries(GetDirectoryName(), NULL, &directories) != INFO::OK)
+        return replays;
+
+    bool newReplays = false;
+    std::vector<u32> copyFromOldCache;
+    // Specifies where the next replay data should go in replays
+    u32 i = 0;
+    {
+        TIMER(L"ReloadReplayCache: Load data");
+        for (const OsPath& directory : directories)
         {
             if (SDL_QuitRequested())
-                return JSVAL_NULL;
+                break;
 
-            JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, directory));
-            if (!replayData.isNull())
+            bool isNew = true;
+            std::map<CStr, std::pair<u32, u32>>::iterator it = fileList.find(directory.string8());
+            // directory is in fileList
+            if (it != fileList.end())
+            {
+                if (compareFileSize)
+                {
+                    CFileInfo fileInfo;
+                    GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo);
+                    if ((u32)fileInfo.Size() == it->second.second)
+                        isNew = false;
+                }
+                else
+                    isNew = false;
+            }
+
+            if (isNew)
+            {
+                JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, directory));
+                if (replayData.isNull())
+                {
+                    CFileInfo fileInfo;
+                    GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo);
+                    scriptInterface.Eval("({})", &replayData);
+                    scriptInterface.SetProperty(replayData, "directory", directory);
+                    scriptInterface.SetProperty(replayData, "fileSize", (u32)fileInfo.Size());
+                }
                 JS_SetElement(cx, replays, i++, replayData);
+                newReplays = true;
+            }
+            else
+                copyFromOldCache.push_back(it->second.first);
         }
-    return JS::ObjectValue(*replays);
+    }
+
+    debug_printf("old not deleted replays: %d \n", copyFromOldCache.size());
+    debug_printf("old deleted replays:: %d \n", fileList.size() - copyFromOldCache.size());
+    debug_printf("new replays: %d \n", i);
+
+    if (!newReplays && fileList.empty())
+        return replays;
+    // No replay was changed, so just return the cache
+    if (!newReplays && fileList.size() == copyFromOldCache.size())
+        return cachedReplaysObject;
+
+    {
+        TIMER(L"ReloadReplayCache: copy from old cache");
+        // Copy the replays from the old cache that are not deleted
+        if (!copyFromOldCache.empty())
+            for (u32 j : copyFromOldCache)
+            {
+                JS::RootedValue replay(cx);
+                JS_GetElement(cx, cachedReplaysObject, j, &replay);
+                JS_SetElement(cx, replays, i++, replay);
+            }
+    }
+
+    {
+        TIMER(L"ReloadReplayCache: save new cache");
+        JS::RootedValue replaysRooted(cx, JS::ObjectValue(*replays));
+        std::ofstream stream(tempCacheFileName.string8().c_str(), std::ofstream::out | std::ofstream::trunc);
+        stream << scriptInterface.StringifyJSON(&replaysRooted);
+        stream.close();
+
+        wunlink(cacheFileName);
+        if (wrename(tempCacheFileName, cacheFileName))
+            LOGERROR("Could not store the replay cache");
+    }
+    return replays;
 }
 
+JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface, bool reload)
+{
+    TIMER(L"GetReplays");
+    JSContext* cx = scriptInterface.GetContext();
+    JSAutoRequest rq(cx);
+    // Acquire replays
+    JS::RootedObject replays(cx, ReloadReplayCache(scriptInterface, reload));
+    // Remove entries without data
+    JS::RootedObject replaysWithoutNullEntries(cx, JS_NewArrayObject(cx, 0));
+    u32 replaysLength = 0;
+    u32 i = 0;
+    JS_GetArrayLength(cx, replays, &replaysLength);
+    for (u32 j = 0; j < replaysLength; ++j)
+    {
+        JS::RootedValue replay(cx);
+        JS_GetElement(cx, replays, j, &replay);
+        if (scriptInterface.HasProperty(replay, "attribs"))
+            JS_SetElement(cx, replaysWithoutNullEntries, i++, replay);
+    }
+    return JS::ObjectValue(*replaysWithoutNullEntries);
+}
+
 /**
  * Move the cursor backwards until a newline was read or the beginning of the file was found.
  * Either way the cursor points to the beginning of a newline.
@@ -173 +305 @@
     return -1;
 }
 
-JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory)
+JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory)
 {
+    TIMER(L"LoadReplayData");
     // The directory argument must not be constant, otherwise concatenating will fail
     const OsPath replayFile = GetDirectoryName() / directory / L"commands.txt";
 
@@ -251 +384 @@
     scriptInterface.Eval("({})", &replayData);
     scriptInterface.SetProperty(replayData, "file", replayFile);
     scriptInterface.SetProperty(replayData, "directory", directory);
+    scriptInterface.SetProperty(replayData, "fileSize", (u32)fileSize);
     scriptInterface.SetProperty(replayData, "filemod_timestamp", std::to_string(fileTime));
     scriptInterface.SetProperty(replayData, "attribs", attribs);
     scriptInterface.SetProperty(replayData, "duration", duration);
@@ -266 +400 @@
     return DirectoryExists(directory) && DeleteDirectory(directory) == INFO::OK;
 }
 
-
 JS::Value VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
 {
     // Create empty JS object
@@ -292 +425 @@
     return attribs;
 }
 
+void VisualReplay::AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName)
+{
+    TIMER(L"AddReplayToCache");
+    JSContext* cx = scriptInterface.GetContext();
+    JSAutoRequest rq(cx);
+
+    JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, OsPath(directoryName)));
+
+    if (replayData.isNull())
+        return;
+
+    const OsPath cacheFileName = GetDirectoryName() / L"replayCache.json";
+    JS::RootedObject cachedReplaysObject(cx, JS_NewArrayObject(cx, 0));
+
+    if (FileExists(cacheFileName))
+    {
+        // Open cache file
+        std::istream* cacheStream = new std::ifstream(cacheFileName.string8().c_str());
+
+        // Read file into chacheStr
+        CStr cacheStr((std::istreambuf_iterator<char>(*cacheStream)), std::istreambuf_iterator<char>());
+
+        // Create empty JS object and parse the content of the cache into it
+        JS::RootedValue cachedReplays(cx);
+        scriptInterface.ParseJSON(cacheStr, &cachedReplays);
+        SAFE_DELETE(cacheStream);
+
+        cachedReplaysObject = &cachedReplays.toObject();
+    }
+
+    u32 cacheLength = 0;
+    JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength);
+    JS_SetElement(cx, cachedReplaysObject, cacheLength, replayData);
+
+    JS::RootedValue replaysRooted(cx, JS::ObjectValue(*cachedReplaysObject));
+    std::ofstream cacheStream(cacheFileName.string8().c_str(), std::ofstream::out | std::ofstream::trunc);
+    cacheStream << scriptInterface.StringifyJSON(&replaysRooted);
+    cacheStream.close();
+}
+
 void VisualReplay::SaveReplayMetadata(ScriptInterface* scriptInterface)
 {
     JSContext* cx = scriptInterface->GetContext();

source/ps/VisualReplay.h

@@ -42 +42 @@
 void StartVisualReplay(const CStrW& directory);
 
 /**
+ * Reloads the replay cache
+ *
+ * @param scriptInterface the ScriptInterface in which to create the return data.
+ * @param compareFileSize compare the directory name and the FileSize of the replays and the cache
+ * @return cache entries
+ */
+JS::HandleObject ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFileSize = false);
+
+/**
  * Get a list of replays to display in the GUI.
  *
  * @param scriptInterface the ScriptInterface in which to create the return data.
  * @return array of objects containing replay data
  */
-JS::Value GetReplays(ScriptInterface& scriptInterface);
+JS::Value GetReplays(ScriptInterface& scriptInterface, bool reload);
 
 /**
  * Parses a commands.txt file and extracts metadata.
  * Works similarly to CGame::LoadReplayData().
  */
-JS::Value LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory);
+JS::Value LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory);
 
 /**
  * Permanently deletes the visual replay (including the parent directory)
@@ -83 +92 @@
  */
 void SaveReplayMetadata(ScriptInterface* scriptInterface);
 
+/**
+* Adds a replay to the replayCache
+*/
+void AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName);
 }
 
 #endif