Ticket #3433: 3433_replay_cache_v1.8.1patch

Index: binaries/data/mods/public/gui/replaymenu/replay_actions.js
===================================================================
--- binaries/data/mods/public/gui/replaymenu/replay_actions.js  (revision 19041)
+++ binaries/data/mods/public/gui/replaymenu/replay_actions.js  (working copy)
@@ -127,6 +127,12 @@
    });
 }

+function reloadCache()
+{
+   let selected = Engine.GetGUIObjectByName("replaySelection").selected;
+   loadReplays(selected > 0 ? createReplaySelectionData(g_ReplaysFiltered[selected].directory) : "", true);
+}
+
 /**
  * Callback.
  */
Index: binaries/data/mods/public/gui/replaymenu/replay_menu.js
===================================================================
--- binaries/data/mods/public/gui/replaymenu/replay_menu.js (revision 19041)
+++ binaries/data/mods/public/gui/replaymenu/replay_menu.js (working copy)
@@ -59,7 +59,7 @@
        return;
    }

-   loadReplays(data && data.replaySelectionData);
+   loadReplays(data && data.replaySelectionData, false);

    if (!g_Replays)
    {
@@ -76,9 +76,9 @@
  * Check timestamp and compatibility and extract g_Playernames, g_MapNames, g_VictoryConditions.
  * Restore selected filters and item.
  */
-function loadReplays(replaySelectionData)
+function loadReplays(replaySelectionData, reload)
 {
-   g_Replays = Engine.GetReplays();
+   g_Replays = Engine.GetReplays(reload);

    if (!g_Replays)
        return;
Index: binaries/data/mods/public/gui/replaymenu/replay_menu.xml
===================================================================
--- binaries/data/mods/public/gui/replaymenu/replay_menu.xml    (revision 19041)
+++ binaries/data/mods/public/gui/replaymenu/replay_menu.xml    (working copy)
@@ -250,6 +250,12 @@
                <translatableAttribute id="caption">Delete</translatableAttribute>
                <action on="Press">deleteReplayButtonPressed();</action>
            </object>
+
+           <!-- Reload Cache Button -->
+           <object type="button" style="StoneButton" size="40%+25 0 57%+25 100%">
+               <translatableAttribute id="caption">Reload Cache</translatableAttribute>
+               <action on="Press">reloadCache();</action>
+           </object>

            <!-- Summary Button -->
            <object name="summaryButton" type="button" style="StoneButton" size="65%-50 0 82%-50 100%">
Index: binaries/data/mods/public/gui/session/session.js
===================================================================
--- binaries/data/mods/public/gui/session/session.js    (revision 19041)
+++ binaries/data/mods/public/gui/session/session.js    (working copy)
@@ -583,6 +583,11 @@

    Engine.EndGame();

+   // After the replay file was closed in EndGame
+   // Done here to keep EndGame small
+   if (!g_IsReplay)
+       Engine.AddReplayToCache(replayDirectory);
+
    if (g_IsController && Engine.HasXmppClient())
        Engine.SendUnregisterGame();

Index: source/ps/scripting/JSInterface_VisualReplay.cpp
===================================================================
--- source/ps/scripting/JSInterface_VisualReplay.cpp    (revision 19041)
+++ source/ps/scripting/JSInterface_VisualReplay.cpp    (working copy)
@@ -33,9 +33,9 @@
    return VisualReplay::DeleteReplay(replayFile);
 }

-JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate)
+JS::Value JSI_VisualReplay::GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool reload)
 {
-   return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface));
+   return VisualReplay::GetReplays(*(pCxPrivate->pScriptInterface), reload);
 }

 JS::Value JSI_VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
@@ -53,12 +53,18 @@
    return VisualReplay::GetReplayMetadata(pCxPrivate, directoryName);
 }

+void JSI_VisualReplay::AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
+{
+   VisualReplay::AddReplayToCache(*(pCxPrivate->pScriptInterface), directoryName);
+}
+
 void JSI_VisualReplay::RegisterScriptFunctions(ScriptInterface& scriptInterface)
 {
-   scriptInterface.RegisterFunction<JS::Value, &GetReplays>("GetReplays");
+   scriptInterface.RegisterFunction<JS::Value, bool, &GetReplays>("GetReplays");
    scriptInterface.RegisterFunction<bool, CStrW, &DeleteReplay>("DeleteReplay");
    scriptInterface.RegisterFunction<void, CStrW, &StartVisualReplay>("StartVisualReplay");
    scriptInterface.RegisterFunction<JS::Value, CStrW, &GetReplayAttributes>("GetReplayAttributes");
    scriptInterface.RegisterFunction<JS::Value, CStrW, &GetReplayMetadata>("GetReplayMetadata");
    scriptInterface.RegisterFunction<bool, CStrW, &HasReplayMetadata>("HasReplayMetadata");
+   scriptInterface.RegisterFunction<void, CStrW, &AddReplayToCache>("AddReplayToCache");
 }
Index: source/ps/scripting/JSInterface_VisualReplay.h
===================================================================
--- source/ps/scripting/JSInterface_VisualReplay.h  (revision 19041)
+++ source/ps/scripting/JSInterface_VisualReplay.h  (working copy)
@@ -25,10 +25,11 @@
 {
    void StartVisualReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directory);
    bool DeleteReplay(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& replayFile);
-   JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate);
+   JS::Value GetReplays(ScriptInterface::CxPrivate* pCxPrivate, bool reload);
    JS::Value GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
    bool HasReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
    JS::Value GetReplayMetadata(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
+   void AddReplayToCache(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName);
    void RegisterScriptFunctions(ScriptInterface& scriptInterface);
 }

Index: source/ps/VisualReplay.cpp
===================================================================
--- source/ps/VisualReplay.cpp  (revision 19041)
+++ source/ps/VisualReplay.cpp  (working copy)
@@ -61,35 +61,196 @@
    g_Game->StartVisualReplay(replayFile.string8());
 }

+bool VisualReplay::ReadCacheFile(ScriptInterface& scriptInterface, JS::MutableHandleObject cachedReplaysObject)
+{
+   TIMER(L"ReadCacheFile");
+   JSContext* cx = scriptInterface.GetContext();
+   JSAutoRequest rq(cx);
+
+   const OsPath cacheFileName = GetDirectoryName() / L"replayCache.json";
+   if (!FileExists(cacheFileName))
+       return false;
+
+   // Open cache file
+   std::ifstream cacheStream(cacheFileName.string8().c_str());
+   // Read file into cacheStr
+   CStr cacheStr((std::istreambuf_iterator<char>(cacheStream)), std::istreambuf_iterator<char>());
+   cacheStream.close();
+
+   // Create empty JS object and parse the context of the cache into it
+   JS::RootedValue cachedReplays(cx);
+   if (!scriptInterface.ParseJSON(cacheStr, &cachedReplays))
+   {
+       LOGERROR("The replay cache file is corrupted, it will be deleted");
+       wunlink(cacheFileName);
+       return false;
+   }
+
+   cachedReplaysObject.set(&cachedReplays.toObject());
+   if (!JS_IsArrayObject(cx, cachedReplaysObject))
+   {
+       LOGERROR("The replay cache file is corrupted, it will be deleted");
+       wunlink(cacheFileName);
+       return false;
+   }
+
+   return true;
+}
+
+void VisualReplay::StoreCacheFile(ScriptInterface& scriptInterface, JS::HandleObject replays)
+{
+   TIMER(L"StoreCacheFile");
+   JSContext* cx = scriptInterface.GetContext();
+   JSAutoRequest rq(cx);
+
+   const OsPath cacheFileName = GetDirectoryName() / L"replayCache.json";
+   const OsPath tempCacheFileName = GetDirectoryName() / L"replayCache_temp.json";
+
+   JS::RootedValue replaysRooted(cx, JS::ObjectValue(*replays));
+   std::ofstream cacheStream(tempCacheFileName.string8().c_str(), std::ofstream::out | std::ofstream::trunc);
+   cacheStream << scriptInterface.StringifyJSON(&replaysRooted);
+   cacheStream.close();
+
+   wunlink(cacheFileName);
+   if (wrename(tempCacheFileName, cacheFileName))
+       LOGERROR("Could not store the replay cache");
+}
+
 /**
- * Load all replays found in the directory.
- *
- * Since files are spread across the harddisk,
- * loading hundreds of them can consume a lot of time.
- */
-JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface)
+* Load the replay cache and check if there are new/deleted ones
+* If so, update their data.
+*/
+JS::HandleObject VisualReplay::ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFileSize)
 {
-   TIMER(L"GetReplays");
+   TIMER(L"ReloadReplayCache");
    JSContext* cx = scriptInterface.GetContext();
    JSAutoRequest rq(cx);
+   // Maps the filename onto the index and size
+   std::map<CStr, std::pair<u32, u32>> fileList;

-   u32 i = 0;
+   JS::RootedObject cachedReplaysObject(cx);
+
+
+   if (ReadCacheFile(scriptInterface, &cachedReplaysObject))
+   {
+       TIMER(L"ReloadReplayCache: create fileList");
+       // Create list of files included in the cache
+       u32 cacheLength = 0;
+       JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength);
+       for (u32 j = 0; j < cacheLength; ++j)
+       {
+           JS::RootedValue replay(cx);
+           JS_GetElement(cx, cachedReplaysObject, j, &replay);
+
+           JS::RootedValue file(cx);
+           CStr fileName;
+           u32 fileSize;
+           scriptInterface.GetProperty(replay, "directory", fileName);
+           scriptInterface.GetProperty(replay, "fileSize", fileSize);
+
+           fileList.emplace(fileName, std::make_pair(j, fileSize));
+       }
+   }
+
+   JS::RootedObject replays(cx, JS_NewArrayObject(cx, 0));
    DirectoryNames directories;
-   JS::RootedObject replays(cx, JS_NewArrayObject(cx, 0));

-   if (GetDirectoryEntries(GetDirectoryName(), NULL, &directories) == INFO::OK)
-       for (OsPath& directory : directories)
+   if (GetDirectoryEntries(GetDirectoryName(), NULL, &directories) != INFO::OK)
+       return replays;
+
+   bool newReplays = false;
+   std::vector<u32> copyFromOldCache;
+   // Specifies where the next replay data should go in replays
+   u32 i = 0;
+   {
+       TIMER(L"ReloadReplayCache: Load data");
+       for (const OsPath& directory : directories)
        {
            if (SDL_QuitRequested())
-               return JSVAL_NULL;
+               return replays;

-           JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, directory));
-           if (!replayData.isNull())
+           bool isNew = true;
+           std::map<CStr, std::pair<u32, u32>>::iterator it = fileList.find(directory.string8());
+           // directory is in fileList
+           if (it != fileList.end())
+           {
+               if (compareFileSize)
+               {
+                   CFileInfo fileInfo;
+                   GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo);
+                   if ((u32)fileInfo.Size() == it->second.second)
+                       isNew = false;
+               }
+               else
+                   isNew = false;
+           }
+
+           if (isNew)
+           {
+               JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, directory));
+               if (replayData.isNull())
+               {
+                   CFileInfo fileInfo;
+                   GetFileInfo(GetDirectoryName() / directory / L"commands.txt", &fileInfo);
+                   scriptInterface.Eval("({})", &replayData);
+                   scriptInterface.SetProperty(replayData, "directory", directory);
+                   scriptInterface.SetProperty(replayData, "fileSize", (u32)fileInfo.Size());
+               }
                JS_SetElement(cx, replays, i++, replayData);
+               newReplays = true;
+           }
+           else
+               copyFromOldCache.push_back(it->second.first);
        }
-   return JS::ObjectValue(*replays);
+   }
+
+   debug_printf("old not deleted replays: %d \n", copyFromOldCache.size());
+   debug_printf("old deleted replays:: %d \n", fileList.size() - copyFromOldCache.size());
+   debug_printf("new replays: %d \n", i);
+
+   if (!newReplays && fileList.empty())
+       return replays;
+   // No replay was changed, so just return the cache
+   if (!newReplays && fileList.size() == copyFromOldCache.size())
+       return cachedReplaysObject;
+
+   {
+       TIMER(L"ReloadReplayCache: copy from old cache");
+       // Copy the replays from the old cache that are not deleted
+       if (!copyFromOldCache.empty())
+           for (u32 j : copyFromOldCache)
+           {
+               JS::RootedValue replay(cx);
+               JS_GetElement(cx, cachedReplaysObject, j, &replay);
+               JS_SetElement(cx, replays, i++, replay);
+           }
+   }
+   StoreCacheFile(scriptInterface, replays);
+   return replays;
 }

+JS::Value VisualReplay::GetReplays(ScriptInterface& scriptInterface, bool reload)
+{
+   TIMER(L"GetReplays");
+   JSContext* cx = scriptInterface.GetContext();
+   JSAutoRequest rq(cx);
+   // Acquire replays
+   JS::RootedObject replays(cx, ReloadReplayCache(scriptInterface, reload));
+   // Remove entries without data
+   JS::RootedObject replaysWithoutNullEntries(cx, JS_NewArrayObject(cx, 0));
+   u32 replaysLength = 0;
+   u32 i = 0;
+   JS_GetArrayLength(cx, replays, &replaysLength);
+   for (u32 j = 0; j < replaysLength; ++j)
+   {
+       JS::RootedValue replay(cx);
+       JS_GetElement(cx, replays, j, &replay);
+       if (scriptInterface.HasProperty(replay, "attribs"))
+           JS_SetElement(cx, replaysWithoutNullEntries, i++, replay);
+   }
+   return JS::ObjectValue(*replaysWithoutNullEntries);
+}
+
 /**
  * Move the cursor backwards until a newline was read or the beginning of the file was found.
  * Either way the cursor points to the beginning of a newline.
@@ -173,8 +334,9 @@
    return -1;
 }

-JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory)
+JS::Value VisualReplay::LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory)
 {
+   TIMER(L"LoadReplayData");
    // The directory argument must not be constant, otherwise concatenating will fail
    const OsPath replayFile = GetDirectoryName() / directory / L"commands.txt";

@@ -251,6 +413,7 @@
    scriptInterface.Eval("({})", &replayData);
    scriptInterface.SetProperty(replayData, "file", replayFile);
    scriptInterface.SetProperty(replayData, "directory", directory);
+   scriptInterface.SetProperty(replayData, "fileSize", (u32)fileSize);
    scriptInterface.SetProperty(replayData, "filemod_timestamp", std::to_string(fileTime));
    scriptInterface.SetProperty(replayData, "attribs", attribs);
    scriptInterface.SetProperty(replayData, "duration", duration);
@@ -266,7 +429,6 @@
    return DirectoryExists(directory) && DeleteDirectory(directory) == INFO::OK;
 }

-
 JS::Value VisualReplay::GetReplayAttributes(ScriptInterface::CxPrivate* pCxPrivate, const CStrW& directoryName)
 {
    // Create empty JS object
@@ -292,6 +454,27 @@
    return attribs;
 }

+void VisualReplay::AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName)
+{
+   TIMER(L"AddReplayToCache");
+   JSContext* cx = scriptInterface.GetContext();
+   JSAutoRequest rq(cx);
+
+   JS::RootedValue replayData(cx, LoadReplayData(scriptInterface, OsPath(directoryName)));
+   if (replayData.isNull())
+       return;
+
+   JS::RootedObject cachedReplaysObject(cx);
+   if (!ReadCacheFile(scriptInterface, &cachedReplaysObject))
+       cachedReplaysObject = JS_NewArrayObject(cx, 0);
+
+   u32 cacheLength = 0;
+   JS_GetArrayLength(cx, cachedReplaysObject, &cacheLength);
+   JS_SetElement(cx, cachedReplaysObject, cacheLength, replayData);
+
+   StoreCacheFile(scriptInterface, cachedReplaysObject);
+}
+
 void VisualReplay::SaveReplayMetadata(ScriptInterface* scriptInterface)
 {
    JSContext* cx = scriptInterface->GetContext();
Index: source/ps/VisualReplay.h
===================================================================
--- source/ps/VisualReplay.h    (revision 19041)
+++ source/ps/VisualReplay.h    (working copy)
@@ -42,18 +42,44 @@
 void StartVisualReplay(const CStrW& directory);

 /**
+ * Reads the replay Cache file and parses it into a jsObject
+ *
+ * @param scriptInterface the ScriptInterface in which to create the return data.
+ * @param cachedReplaysObject the cached replays
+ * @return true on succes; false otherwise
+ */
+bool ReadCacheFile(ScriptInterface& scriptInterface, JS::MutableHandleObject cachedReplaysObject);
+
+/**
+ * Stores the replay list in the replay cache file
+ *
+ * @param scriptInterface the ScriptInterface in which to create the return data.
+ * @param replays teh replay list to store
+ */
+void StoreCacheFile(ScriptInterface& scriptInterface, JS::HandleObject replays);
+
+/**
+ * Reloads the replay cache
+ *
+ * @param scriptInterface the ScriptInterface in which to create the return data.
+ * @param compareFileSize compare the directory name and the FileSize of the replays and the cache
+ * @return cache entries
+ */
+JS::HandleObject ReloadReplayCache(ScriptInterface& scriptInterface, bool compareFileSize = false);
+
+/**
  * Get a list of replays to display in the GUI.
  *
  * @param scriptInterface the ScriptInterface in which to create the return data.
  * @return array of objects containing replay data
  */
-JS::Value GetReplays(ScriptInterface& scriptInterface);
+JS::Value GetReplays(ScriptInterface& scriptInterface, bool reload);

 /**
  * Parses a commands.txt file and extracts metadata.
  * Works similarly to CGame::LoadReplayData().
  */
-JS::Value LoadReplayData(ScriptInterface& scriptInterface, OsPath& directory);
+JS::Value LoadReplayData(ScriptInterface& scriptInterface, const OsPath& directory);

 /**
  * Permanently deletes the visual replay (including the parent directory)
@@ -83,6 +109,10 @@
  */
 void SaveReplayMetadata(ScriptInterface* scriptInterface);

+/**
+* Adds a replay to the replayCache
+*/
+void AddReplayToCache(ScriptInterface& scriptInterface, const CStrW& directoryName);
 }

 #endif