Ticket #1396: track1396.patch

binaries/data/mods/_test.sim/simulation/components/test-bom-none.js

@@ +1 @@
+/* A simple JavaScript file in UTF-8 without byte order mark
+   at the start of the file, to ensure such scripts are handled
+   properly. */
+Engine.Note = "Script without BOM was accepted";

binaries/data/mods/_test.sim/simulation/components/test-bom-utf8.js

@@ +1 @@
+/* A simple JavaScript file in UTF-8 with byte order mark/ at the start
+   of the file (probably not visible in a regular editor), to ensure such
+   scripts are handled properly.*/
+
+Engine.Note = "UTF8-BOM was accepted"; /* ID to ensure
+    the function was really evaluated ☺ */

source/gui/CGUI.h

@@ -421 +421 @@
     void Xeromyces_ReadRootSetup(XMBElement Element, CXeromyces* pFile);
 
     // Read Subs
-
+protected: // Members protected so unit tests can access them via inheritance
+    
     /**
      * Notice! Recursive function!
      *

source/gui/tests/test_CGUI.h

@@ +1 @@
+/* Copyright (C) 2014 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "gui/CGUI.h"
+#include "gui/IGUIObject.h"
+#include "lib/utf8.h"
+#include "lib/file/file.h"
+#include "lib/file/vfs/vfs.h"
+#include "scriptinterface/ScriptInterface.h"
+#include "ps/CLogger.h"
+#include "ps/Filesystem.h"
+
+/**
+ * Helper namespace to keep utility classes for testing of CGUI, namely
+ * the VFS fakes (prevent collision with other tests)
+ */
+namespace test_CGUI
+{
+    /**
+     * Lightweight testing implementation of a virtual file system.
+     *
+     * While the TestSimulation2 class already has an associated mod (_test.sim),
+     * testing of the file loader routines via a stub has several advantages:
+     *  1.  Some routines, such as GetRMSData(), scan a whole directory whose path is
+     *      hard-coded in the testee. Using a real VFS might cause undesired interferences
+     *      as more tests are added for a single routine.
+     *  2.  A pure in-memory VFS is much faster.
+     *  3.  For UTF-8 tests, control characters (such as a leading BOM) may be expressed
+     *      directly in the C code, while they might be invisible in some editors when
+     *      put in real files.
+     */
+    struct StubVFS : public IVFS {
+
+        /**
+         * Simulates a file.
+         */
+        class StubFile
+        {
+        public:
+
+            /**
+             * Initializes a new stub file with fixed content.
+             * @param filename Name of the file.
+             * @param dataLength Size of the data held by the fake file.
+             * @param data Data found in the associated file.
+             */
+            StubFile(const char filename[], size_t dataLength, const u8 data[]) :
+                m_FileName(filename),
+                m_FileSize(dataLength)
+            {
+                u8 *dataCopy = new u8[dataLength];
+                memcpy(dataCopy, data, dataLength);
+                this->m_FileContent = shared_ptr<u8>(dataCopy);
+            }
+
+            /**
+             * Provides the data size of the file.
+             * @return An info record with file size and a fake date.
+             */
+            CFileInfo getInfo() const
+            {
+                return CFileInfo(this->getName(), this->m_FileSize, 0);
+            }
+
+            /**
+             * Retrieves the name of the file.
+             * @return The filename passed into the constructor.
+             */
+            OsPath getName() const
+            {
+                return OsPath(this->m_FileName);
+            }
+
+            /**
+             * Retrieves the data of this file.
+             * @return A copy of the data passed to the constructor.
+             */
+            shared_ptr<u8> getContent() const
+            {
+                return this->m_FileContent;
+            }
+
+        private:
+
+            /**
+             * Name of this file.
+             */
+            const char *m_FileName;
+
+            /**
+             * Data found in the file represented by this object
+             */
+            shared_ptr<u8> m_FileContent;
+
+            /**
+             * Data size of 'm_FileContent'
+             */
+            const size_t m_FileSize;
+        };
+
+        /**
+         * Creates a stub file system which simulates a fixed set of files.
+         * @param numberOfFiles Number of valid entries in 'files'.
+         * @param files The files which are provided (no real support for
+         *              directories, all files are returned on each call to
+         *              'GetDirectoryEntries').
+         */
+        StubVFS(size_t numberOfFiles, const StubFile files[]) :
+            m_NumberOfFiles(numberOfFiles)
+        {
+            this->m_Files = files;
+        }
+
+        virtual Status Mount(const VfsPath&, const OsPath&, size_t, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFileInfo(const VfsPath &path, CFileInfo *pInfo) const
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (this->m_Files[i].getName() == path.Filename())
+                {
+                    if (pInfo)
+                    {
+                        *pInfo = this->m_Files[i].getInfo();
+                    }
+                    return INFO::OK;
+                }
+            }
+            return ERR::FILE_NOT_FOUND;
+        }
+
+        virtual Status GetFilePriority(const VfsPath &path, size_t *ppriority) const
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (this->m_Files[i].getName() == path.Filename())
+                {
+                    *ppriority = 1;
+                    return INFO::OK;
+                }
+            }
+            return ERR::FILE_NOT_FOUND;
+        }
+
+        /**
+         * Retrieves a listing of all files held by this file system.
+         * @param infos Receives additional information about a file.
+         * @param names Receives the names of all files in the file system.
+         * @return OK.
+         */
+        virtual Status GetDirectoryEntries(const VfsPath&, CFileInfos* infos, DirectoryNames* names) const
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (infos)
+                {
+                    infos->push_back(this->m_Files[i].getInfo());
+                }
+                if (names)
+                {
+                    names->push_back(this->m_Files[i].getName());
+                }
+            }
+            return INFO::OK;
+        }
+
+        virtual Status CreateFile(const VfsPath &, const shared_ptr<u8>&, size_t)
+        {
+            // Currently, test cases do not involve writing files back to disk;
+            // this function is called by Xeromyces in an attempt to cache an XML
+            // document in binary mode
+            return INFO::OK;
+        }
+
+        virtual Status ReplaceFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Read an entire file into memory.
+         *
+         * @param path Selects the fake file whose content is requested.
+         * @param fileContents receives a smart pointer to the contents.
+         *        CAVEAT: this will be taken from the file cache if the VFS was
+         *        created with cacheSize != 0 and size < cacheSize. There is no
+         *        provision for Copy-on-Write, which means that such buffers
+         *        must not be modified (this is enforced via mprotect).
+         * @param size receives the size [bytes] of the file contents.
+         * @return OK, if a fake file with given path was selected, or
+         *         OpenFailed in case the file does not exist.
+         **/
+        virtual Status LoadFile(const VfsPath& path, shared_ptr<u8>& fileContents, size_t& size)
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (this->m_Files[i].getName() == path.Filename())
+                {
+                    fileContents = this->m_Files[i].getContent();
+                    size = this->m_Files[i].getInfo().Size();
+                    return INFO::OK;
+                }
+            }
+            return CODE__PSRETURN_Scripting_LoadFile_OpenFailed;
+        }
+
+        virtual std::wstring TextRepresentation() const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetRealPath(const VfsPath &virtualPath, OsPath &realPath)
+        {
+            realPath = virtualPath;
+            return INFO::OK;
+        }
+
+        virtual Status GetDirectoryRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetVirtualPath(const OsPath&, VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * remove file from the virtual directory listing and evict its
+         * data from the cache.
+         **/
+        virtual Status RemoveFile(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RepopulateDirectory(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual void Clear()
+        {
+            throw "Not yet implemented";
+        }
+
+    private:
+
+        /**
+         * Number of valid elements in 'm_Files'.
+         */
+        const size_t m_NumberOfFiles;
+
+        /**
+         * The files found in this file system.
+         */
+        const StubFile *m_Files;
+    };
+
+    /**
+     * Helper to collect the actual elements loaded from Xeromyces
+     */
+    class MockGUIObject : public IGUIObject
+    {
+    public:
+
+        virtual void Draw(void)
+        {
+            // No need to draw anything, used only for testing
+        }
+
+        /**
+         * Retrieves the first child from the internal list of children.
+         * @return A reference to the first child, NULL if there are no children.
+         */
+        IGUIObject *GetFirstChild()
+        {
+            return this->m_Children.size() > 0 ? this->m_Children[0] : NULL;
+        }
+
+        /**
+         * Helper member for the BOM tests to run the JavaScript handlers registered
+         * by the Xeromyces loader
+         */
+        virtual void HandleMessage(SGUIMessage& UNUSED(Message))
+        {
+            this->ScriptEvent("validatetest");
+        }
+
+    };
+}
+
+class TestCGUI : public CxxTest::TestSuite 
+{
+public:
+
+    class CGUI_TestHook : public CGUI
+    {
+    public:
+        CGUI_TestHook(const shared_ptr<ScriptRuntime>& runtime) : CGUI(runtime)
+        {
+        }
+
+        void Xeromyces_ReadObject(XMBElement Element, CXeromyces* pFile, IGUIObject *pParent, const std::vector<std::pair<CStr, CStr> >& NameSubst, boost::unordered_set<VfsPath>& Paths)
+        {
+            CGUI::Xeromyces_ReadObject(Element, pFile, pParent, NameSubst, Paths);
+        }
+
+    };
+
+
+    void setUp()
+    {
+        CXeromyces::Startup();
+    }
+
+    void tearDown()
+    {
+        CXeromyces::Terminate();
+    }   
+
+    static IGUIObject *CreateMockObject()
+    {
+        return new test_CGUI::MockGUIObject();
+    }
+
+    /**
+     * An action handler (external JavaScript file) is loaded when stored as a
+     * raw ASCII file.
+     */
+    void test_Xeromyces_ReadObject_RawASCIISupportedAsActionScript()
+    {
+        const u8 xmlData[] = "\
+<object type=\"testobj\">\
+    <action file=\"test.js\" on=\"validatetest\"/>\
+</object>";
+        const u8 jsData[] = "warn(\"Raw ASCII accepted!\");";
+
+        test_CGUI::StubVFS::StubFile testFiles[] =
+        {
+            test_CGUI::StubVFS::StubFile("test.xml", sizeof(xmlData)/sizeof(xmlData[0]) - 1, xmlData),
+            test_CGUI::StubVFS::StubFile("test.js", sizeof(jsData)/sizeof(jsData[0]) - 1, jsData)
+        };
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_CGUI::StubVFS(2, testFiles));
+
+        CXeromyces testObjDescription;
+        PSRETURN loadingResult = testObjDescription.Load(g_VFS, "test.xml");
+
+        TS_ASSERT_EQUALS(loadingResult, INFO::OK);
+
+        XMBElement rootNode = testObjDescription.GetRoot();
+
+        CGUI_TestHook testee(ScriptInterface::CreateRuntime());
+        std::vector<std::pair<CStr8, CStr8> > nameSubst;
+        boost::unordered_set<VfsPath> vfsPaths;
+        test_CGUI::MockGUIObject mockParent;
+
+        testee.AddObjectType("testobj", &CreateMockObject);
+
+        testee.Xeromyces_ReadObject(
+            rootNode,
+            &testObjDescription,
+            &mockParent,
+            nameSubst,
+            vfsPaths);
+
+        IGUIObject *pGeneratedChild = mockParent.GetFirstChild();
+
+        // If the test setup above is successfull, pGeneratedChild
+        // is actually a MockGUIObject and its HandleMessage implementation
+        // will run the JavaScript code.
+        TS_ASSERT(NULL != pGeneratedChild);
+
+        TestLogger log;
+
+        SGUIMessage message(GUIM_MOUSE_OVER);
+        pGeneratedChild->HandleMessage(message);
+
+        TS_ASSERT_WSTR_CONTAINS(log.GetOutput(), L"Raw ASCII accepted!");
+
+        g_VFS = originalVFS;
+    }
+
+    /**
+     * An action handler (external JavaScript file) is loaded when stored with
+     * an UTF-8 byte-order-mark at the start of the script code, leaving valid
+     * JavaScript.
+     */
+    void test_Xeromyces_ReadObject_UTF8SupportedAsActionScript()
+    {
+        const u8 xmlData[] = "\
+<object type=\"testobj\">\
+    <action file=\"test.js\" on=\"validatetest\"/>\
+</object>";
+        const u8 jsData[] = "\xEF\xBB\xBFwarn(\"UTF8-BOM accepted!\");";
+
+        test_CGUI::StubVFS::StubFile testFiles[] =
+        {
+            test_CGUI::StubVFS::StubFile("test.xml", sizeof(xmlData)/sizeof(xmlData[0]) - 1, xmlData),
+            test_CGUI::StubVFS::StubFile("test.js", sizeof(jsData)/sizeof(jsData[0]) - 1, jsData)
+        };
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_CGUI::StubVFS(2, testFiles));
+
+        CXeromyces testObjDescription;
+        PSRETURN loadingResult = testObjDescription.Load(g_VFS, "test.xml");
+
+        TS_ASSERT_EQUALS(loadingResult, INFO::OK);
+
+        XMBElement rootNode = testObjDescription.GetRoot();
+
+        CGUI_TestHook testee(ScriptInterface::CreateRuntime());
+        std::vector<std::pair<CStr8, CStr8> > nameSubst;
+        boost::unordered_set<VfsPath> vfsPaths;
+        test_CGUI::MockGUIObject mockParent;
+
+        testee.AddObjectType("testobj", &CreateMockObject);
+
+        testee.Xeromyces_ReadObject(
+            rootNode,
+            &testObjDescription,
+            &mockParent,
+            nameSubst,
+            vfsPaths);
+
+        IGUIObject *pGeneratedChild = mockParent.GetFirstChild();
+
+        // If the test setup above is successfull, pGeneratedChild
+        // is actually a MockGUIObject and its HandleMessage implementation
+        // will run the JavaScript code
+        TS_ASSERT(NULL != pGeneratedChild);
+
+        TestLogger log;
+
+        SGUIMessage message(GUIM_MOUSE_OVER);
+        pGeneratedChild->HandleMessage(message);
+
+        TS_ASSERT_WSTR_CONTAINS(log.GetOutput(), L"UTF8-BOM accepted!");
+
+        g_VFS = originalVFS;
+    }
+
+};

source/ps/GameSetup/HWDetect.cpp

@@ -186 +186 @@
     g_RenderPath = renderpath;
 }
 
-void RunHardwareDetection()
+/**
+ * Loads and initializes the JavaScript library used for hardware
+ * detection.
+ * @param scriptInterface The runtime used for the JS part of the HW detection.
+ * @return true if and only if the script was loaded successfully.
+ */
+bool LoadHardwareDetectionLibrary(ScriptInterface &scriptInterface)
 {
-    TIMER(L"RunHardwareDetection");
-
-    ScriptInterface scriptInterface("Engine", "HWDetect", g_ScriptRuntime);
-
     scriptInterface.RegisterFunction<void, bool, &SetDisableAudio>("SetDisableAudio");
     scriptInterface.RegisterFunction<void, bool, &SetDisableS3TC>("SetDisableS3TC");
     scriptInterface.RegisterFunction<void, bool, &SetDisableShadows>("SetDisableShadows");
@@ -208 +210 @@
     if (file.Load(g_VFS, scriptName) != PSRETURN_OK)
     {
         LOGERROR(L"Failed to load hardware detection script");
-        return;
+        return false;
     }
 
     std::string code = file.DecodeUTF8(); // assume it's UTF-8
-    scriptInterface.LoadScript(scriptName, code);
+    return scriptInterface.LoadScript(scriptName, code);
+}
 
+void RunHardwareDetection()
+{
+    TIMER(L"RunHardwareDetection");
+
+    ScriptInterface scriptInterface("Engine", "HWDetect", g_ScriptRuntime);
+
+    if (LoadHardwareDetectionLibrary(scriptInterface))
+    {
+        return;
+    }
+
     // Collect all the settings we'll pass to the script:
     // (We'll use this same data for the opt-in online reporting system, so it
     // includes some fields that aren't directly useful for the hwdetect script)

source/ps/GameSetup/tests/test_HWDetect.h

@@ +1 @@
+/* Copyright (C) 2010 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "lib/file/vfs/vfs.h"
+#include "scriptinterface/ScriptInterface.h"
+#include "ps/Filesystem.h"
+#include "ps/GameSetup/Config.h"
+#include "ps/GameSetup/HWDetect.h"
+
+/**
+ * Holds utility classes for testing of the HWDetection routines
+ *
+ * Multiple classes use stub implementation of the IVFS
+ * interface; to prevent name collisions among them, the stub is
+ * encapsulated in its own namespace.
+ */
+namespace test_HWDetect
+{
+
+    /**
+     * Lightweight testing implementation of a virtual file system.
+     */
+    struct StubVFS : public IVFS {
+
+        /**
+         * Creates a stub file system which returns a fixed file content for any file.
+         * @param dataLength Number of valid consecutive data bytes in 'fileContent'.
+         * @param fileContent "Data" of all files in the stub file system.
+         */
+        StubVFS(size_t dataLength, const u8 fileContent[]) : m_FileSize(dataLength)
+        {
+            // As the IVFS is defined to return a smart pointer to the data -
+            // see LoadFile() - we have to create a copy of the supplied primitive
+            // memory block, getting rid of the const at the same time.
+
+            u8 *dataCopy = new u8[dataLength];
+            memcpy(dataCopy, fileContent, dataLength);
+            this->m_FileContent = shared_ptr<u8>(dataCopy);
+        }
+
+        virtual Status Mount(const VfsPath&, const OsPath&, size_t, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFileInfo(const VfsPath&, CFileInfo*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFilePriority(const VfsPath&, size_t*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryEntries(const VfsPath&, CFileInfos*, DirectoryNames*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status CreateFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status ReplaceFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Read an entire file into memory.
+         *
+         * @param fileContents receives a smart pointer to the contents.
+         *        CAVEAT: this will be taken from the file cache if the VFS was
+         *        created with cacheSize != 0 and size < cacheSize. There is no
+         *        provision for Copy-on-Write, which means that such buffers
+         *        must not be modified (this is enforced via mprotect).
+         * @param size receives the size [bytes] of the file contents.
+         * @return Status.
+         **/
+        virtual Status LoadFile(const VfsPath&, shared_ptr<u8>& fileContents, size_t& size)
+        {
+            fileContents = this->m_FileContent;
+            size = this->m_FileSize;
+            return INFO::OK;
+        }
+
+        virtual std::wstring TextRepresentation() const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetVirtualPath(const OsPath&, VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RemoveFile(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RepopulateDirectory(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual void Clear()
+        {
+            throw "Not yet implemented";
+        }
+
+    private:
+
+        /**
+         * Data found in all "files" of this file system.
+         */
+        shared_ptr<u8> m_FileContent;
+
+        /**
+         * Number of valid elements in 'm_FileContent'
+         */
+        const size_t m_FileSize;
+    };
+
+}
+
+// In theory, the loader function should not be visible outside HWDetect.cpp,
+// but since that file is procedural, the usual trick of declaring friend with
+// the testee cannot be used to gain access to the function tested. As a
+// compromise, the prototype is declared in the test only.
+extern bool LoadHardwareDetectionLibrary(ScriptInterface &scriptInterface);
+
+class TestHWDetect : public CxxTest::TestSuite
+{
+public:
+
+    /**
+     * The hardware detection JavaScript used by the HWDetect sub-module is accepted
+     * when it contains a lead-in byte order mark.
+     */
+    void test_LoadHWDetection_accepts_BOM()
+    {
+        static const u8 detectionScript[] = "\xEF\xBB\xBF Engine.SetRenderPath(\"BOM accepted\");";
+        PIVFS originalVFS(g_VFS);
+        CStr originalRenderPath(g_RenderPath);
+
+        g_VFS =
+            PIVFS(new test_HWDetect::StubVFS(
+                sizeof(detectionScript)/sizeof(detectionScript[0]) - 1, // -1 as JS does not accept \0
+                detectionScript));
+
+        shared_ptr<ScriptRuntime> runtime = ScriptInterface::CreateRuntime();
+        ScriptInterface scriptInterface("Engine", "HWDetect", runtime);
+
+        LoadHardwareDetectionLibrary(scriptInterface);
+
+        TS_ASSERT_EQUALS(g_RenderPath, "BOM accepted");
+
+        g_VFS = originalVFS;
+        g_RenderPath = originalRenderPath;
+    }
+
+    /**
+     * The hardware detection JavaScript used by the HWDetect.cpp sub-module is accepted
+     * when it does not start with a byte-order-mark (e.g. a plain ASCII file)
+     */
+    void test_LoadHWDetection_accepts_without_BOM()
+    {
+        static const u8 detectionScript[] = "Engine.SetRenderPath(\"ASCII accepted\");";
+        PIVFS originalVFS(g_VFS);
+        CStr originalRenderPath(g_RenderPath);
+
+        g_VFS =
+            PIVFS(new test_HWDetect::StubVFS(
+                sizeof(detectionScript)/sizeof(detectionScript[0]) - 1, // -1 as JS does not accept \0
+                detectionScript));
+
+        shared_ptr<ScriptRuntime> runtime = ScriptInterface::CreateRuntime();
+        ScriptInterface scriptInterface("Engine", "HWDetect", runtime);
+
+        LoadHardwareDetectionLibrary(scriptInterface);
+
+        TS_ASSERT_EQUALS(g_RenderPath, "ASCII accepted");
+
+        g_VFS = originalVFS;
+        g_RenderPath = originalRenderPath;
+    }
+
+};

source/ps/tests/test_CVFSFile.h

@@ +1 @@
+/* Copyright (C) 2012 Wildfire Games.
+ * This file is part of 0 A.D.
+ *
+ * 0 A.D. is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * 0 A.D. is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "precompiled.h"
+#include "ps/Filesystem.h"
+
+/**
+ * Holds utility classes for testing of the CVFSFile class.
+ *
+ * The unit-tests of the CVFSFile use a stub implementation of the IVFS
+ * interface; to prevent name collisions with other tests, the stub is
+ * encapsulated in its own namespace.
+ */
+namespace test_CVFSFile
+{
+
+    /**
+     * Lightweight testing implementation of a virtual file system.
+     */
+    struct StubVFS : public IVFS {
+
+        /**
+         * Creates a stub file system which returns a fixed file content for any file.
+         * @param dataLength Number of valid consecutive data bytes in 'fileContent'.
+         * @param fileContent "Data" of all files in the stub file system.
+         */
+        StubVFS(size_t dataLength, const u8 fileContent[]) : m_FileSize(dataLength)
+        {
+            // As the IVFS is defined to return a smart pointer to the data -
+            // see LoadFile() - we have to create a copy of the supplied primitive
+            // memory block, getting rid of the const at the same time.
+
+            u8 *dataCopy = new u8[dataLength];
+            memcpy(dataCopy, fileContent, dataLength);
+            this->m_FileContent = shared_ptr<u8>(dataCopy);
+        }
+
+        virtual Status Mount(const VfsPath&, const OsPath&, size_t, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFileInfo(const VfsPath&, CFileInfo*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFilePriority(const VfsPath&, size_t*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryEntries(const VfsPath&, CFileInfos*, DirectoryNames*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status CreateFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+
+            throw "Not yet implemented";
+        }
+
+        virtual Status ReplaceFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Read an entire file into memory.
+         *
+         * @param fileContents receives a smart pointer to the contents.
+         *        CAVEAT: this will be taken from the file cache if the VFS was
+         *        created with cacheSize != 0 and size < cacheSize. There is no
+         *        provision for Copy-on-Write, which means that such buffers
+         *        must not be modified (this is enforced via mprotect).
+         * @param size receives the size [bytes] of the file contents.
+         * @return Status.
+         **/
+        virtual Status LoadFile(const VfsPath&, shared_ptr<u8>& fileContents, size_t& size)
+        {
+            fileContents = this->m_FileContent;
+            size = this->m_FileSize;
+            return INFO::OK;
+        }
+
+        virtual std::wstring TextRepresentation() const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetVirtualPath(const OsPath&, VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * remove file from the virtual directory listing and evict its
+         * data from the cache.
+         **/
+        virtual Status RemoveFile(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RepopulateDirectory(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual void Clear()
+        {
+            throw "Not yet implemented";
+        }
+
+    private:
+
+        /**
+         * Data found in all "files" of this file system.
+         */
+        shared_ptr<u8> m_FileContent;
+
+        /**
+         * Number of valid elements in 'm_FileContent'
+         */
+        const size_t m_FileSize;
+    };
+
+}
+
+class TestCVSFFile : public CxxTest::TestSuite
+{
+public:
+
+    /**
+     * A file without an UTF8 byte order mark at the start (e.g. a normal
+     * ASCII file) is loaded properly.
+     */
+    void test_decodeutf8_works_without_bom()
+    {
+        static const unsigned char testContent[] =
+        {
+            'S', 'm', 'i', 'l', 'e', 'y', ' ',
+            ':', '-', ')'
+        };
+
+        PIVFS pvfs =
+            PIVFS(new test_CVFSFile::StubVFS(
+                sizeof(testContent)/sizeof(testContent[0]),
+                testContent));
+
+        CVFSFile testObj;
+
+        PSRETURN state = testObj.Load(pvfs, "Testfile.txt");
+
+        TS_ASSERT_EQUALS(state, PSRETURN_OK);
+
+        CStr decodedStr = testObj.DecodeUTF8();
+
+        TS_ASSERT_EQUALS(decodedStr[0], 'S');
+        TS_ASSERT_EQUALS(decodedStr[2], 'i');
+        TS_ASSERT_EQUALS(decodedStr.size(),
+            sizeof(testContent) / sizeof(testContent[0]));
+    }
+
+    /**
+     * A file with an UTF8 byte order mark at the start is loaded correctly,
+     * and the byte order mark suppressed on decoding.
+     */
+    void test_decodeutf8_accepts_bom()
+    {
+        const size_t BOM_LENGTH = 3;
+        static const unsigned char testContent[] =
+        {
+            0xEF, 0xBB, 0xBF, // Byte order mark sequence
+            'S', 'm', 'i', 'l', 'e', 'y', ' ', // regular text
+            0xE2, 0x98, 0xBA // smiley (263Ah)
+        };
+
+        PIVFS pvfs =
+            PIVFS(new test_CVFSFile::StubVFS(
+                sizeof(testContent)/sizeof(testContent[0]),
+                testContent));
+
+        CVFSFile testObj;
+
+        PSRETURN state = testObj.Load(pvfs, "Testfile.txt");
+
+        TS_ASSERT_EQUALS(state, PSRETURN_OK);
+
+        CStr decodedStr = testObj.DecodeUTF8();
+
+        TS_ASSERT_EQUALS(decodedStr[0], 'S');
+        TS_ASSERT_EQUALS(decodedStr[2], 'i');
+        TS_ASSERT_EQUALS(decodedStr.size(),
+            sizeof(testContent) / sizeof(testContent[0]) - BOM_LENGTH);
+    }
+
+};

source/scriptinterface/tests/test_ScriptInterface.h

@@ -15 +15 @@
  * along with 0 A.D.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "lib/file/vfs/vfs.h"
 #include "lib/self_test.h"
 
 #include "scriptinterface/ScriptInterface.h"
 
 #include "ps/CLogger.h"
+#include "ps/Filesystem.h"
 
 #include <boost/random/linear_congruential.hpp>
 
+/**
+ * Helper namespace to keep utility classes for ScriptInterface unit tests.
+ */
+namespace test_ScriptInterface
+{
+
+    /**
+     * Lightweight testing implementation of a virtual file system. Intended for unit
+     * tests only (escape the need to provide actual files on the disk, and reading
+     * them each time a new test class is launched)
+     */
+    struct StubVFS : public IVFS {
+
+        /**
+         * Creates a stub file system which returns a fixed file content for any file.
+         * @param dataLength Number of valid consecutive data bytes in 'fileContent'.
+         * @param fileContent "Data" of all files in the stub file system.
+         */
+        StubVFS(size_t dataLength, const u8 fileContent[]) : m_FileSize(dataLength)
+        {
+            // As the IVFS is defined to return a smart pointer to the data -
+            // see LoadFile() - we have to create a copy of the supplied primitive
+            // memory block, getting rid of the const at the same time.
+
+            u8 *dataCopy = new u8[dataLength];
+            memcpy(dataCopy, fileContent, dataLength);
+            this->m_FileContent = shared_ptr<u8>(dataCopy);
+        }
+
+        virtual Status Mount(const VfsPath&, const OsPath&, size_t, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Retrieve information about a file (similar to POSIX stat).
+         *
+         * @param pathname
+         * @param pfileInfo receives information about the file. Passing NULL
+         *        suppresses warnings if the file doesn't exist.
+         * 
+         * @return Always OK (any file is faked to exist, and has the content
+         *         supplied in the constructor).
+         **/
+        virtual Status GetFileInfo(const VfsPath& pathname, CFileInfo* pfileInfo) const
+        {
+            if (pfileInfo)
+            {
+                *pfileInfo = CFileInfo(pathname, this->m_FileSize, 0);
+            }
+            return INFO::OK;
+        }
+
+        virtual Status GetFilePriority(const VfsPath&, size_t*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryEntries(const VfsPath&, CFileInfos*, DirectoryNames*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status CreateFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+
+            throw "Not yet implemented";
+        }
+
+        virtual Status ReplaceFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Read an entire file into memory.
+         *
+         * @param fileContents receives a smart pointer to the contents.
+         *        CAVEAT: this will be taken from the file cache if the VFS was
+         *        created with cacheSize != 0 and size < cacheSize. There is no
+         *        provision for Copy-on-Write, which means that such buffers
+         *        must not be modified (this is enforced via mprotect).
+         * @param size receives the size [bytes] of the file contents.
+         * @return Status.
+         **/
+        virtual Status LoadFile(const VfsPath&, shared_ptr<u8>& fileContents, size_t& size)
+        {
+            fileContents = this->m_FileContent;
+            size = this->m_FileSize;
+            return INFO::OK;
+        }
+
+        virtual std::wstring TextRepresentation() const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetVirtualPath(const OsPath&, VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RemoveFile(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RepopulateDirectory(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual void Clear()
+        {
+            throw "Not yet implemented";
+        }
+
+    private:
+
+        /**
+         * Data found in all "files" of this file system.
+         */
+        shared_ptr<u8> m_FileContent;
+
+        /**
+         * Number of valid elements in 'm_FileContent'
+         */
+        const size_t m_FileSize;
+    };
+
+}
+
 class TestScriptInterface : public CxxTest::TestSuite
 {
 public:
+
+    void test_loadglobalscriptfile_accepts_rawascii()
+    {
+        PIVFS originalVFS(g_VFS);
+        ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());
+        TestLogger logger;
+        // Small calculation in output so we can see whether script was really
+        // evaluated
+        static const u8 testScript[] = "warn('The key is: ASC' + (4+7));";
+
+        g_VFS = PIVFS(
+            new test_ScriptInterface::StubVFS(
+                (sizeof(testScript) / sizeof(testScript[0])) - 1,
+                testScript));
+
+        bool scriptLoaded = script.LoadGlobalScriptFile("test.js");
+        TS_ASSERT(scriptLoaded);
+
+        std::wstring scriptOutput = logger.GetOutput();
+        TS_ASSERT_WSTR_CONTAINS(scriptOutput, L"The key is: ASC11");
+        g_VFS = originalVFS;
+    }
+
+    void test_loadglobalscriptfile_accepts_utf8bom()
+    {
+        PIVFS originalVFS(g_VFS);
+        ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());
+        TestLogger logger;
+        // Small calculation in output so we can see whether script was really
+        // evaluated
+        static const u8 testScript[] = "\xEF\xBB\xBFwarn('The key is: ' + (3 * 5));";
+
+        g_VFS = PIVFS(
+            new test_ScriptInterface::StubVFS(
+                (sizeof(testScript) / sizeof(testScript[0])) - 1,
+                testScript));
+
+        bool scriptLoaded = script.LoadGlobalScriptFile("test.js");
+        TS_ASSERT(scriptLoaded);
+
+        std::wstring scriptOutput = logger.GetOutput();
+        TS_ASSERT_WSTR_CONTAINS(scriptOutput, L"The key is: 15");
+        g_VFS = originalVFS;
+    }
+
     void test_loadscript_basic()
     {
         ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());
@@ -126 +315 @@
         TS_ASSERT_EQUALS(d1, d2);
     }
 
+    void test_readjsonfile_accepts_rawascii()
+    {
+        PIVFS originalVFS(g_VFS);
+        static const u8 jsonCode[] = "{ \"encoding\" : \"plain old text\" }";
+        g_VFS = PIVFS(
+            new test_ScriptInterface::StubVFS(
+                sizeof(jsonCode) / sizeof(jsonCode[0]) - 1, // don't include \0 at end
+                jsonCode));
+
+        ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());
+
+        CScriptValRooted jsonData = script.ReadJSONFile("test.json");
+        jsval actualObj = jsonData.get();
+        CStr testValue;
+        script.GetProperty(actualObj, "encoding", testValue);
+
+        TS_ASSERT_EQUALS(testValue, "plain old text");
+
+        g_VFS = originalVFS;
+    }
+
+    void test_readjsonfile_accepts_utf8bom()
+    {
+        PIVFS originalVFS(g_VFS);
+        static const u8 jsonCode[] = "\xEF\xBB\xBF{ \"id\" : \"Herakles\" }";
+        g_VFS = PIVFS(
+            new test_ScriptInterface::StubVFS(
+                sizeof(jsonCode) / sizeof(jsonCode[0]) - 1, // don't include \0 at end
+                jsonCode));
+
+        ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());
+
+        CScriptValRooted jsonData = script.ReadJSONFile("test.json");
+        jsval actualObj = jsonData.get();
+        CStr testValue;
+        script.GetProperty(actualObj, "id", testValue);
+
+        TS_ASSERT_EQUALS(testValue, "Herakles");
+
+        g_VFS = originalVFS;
+    }
+
     void test_json()
     {
         ScriptInterface script("Test", "Test", ScriptInterface::CreateRuntime());

source/simulation2/Simulation2.h

@@ -263 +263 @@
      */
     std::string GetAIData();
 
+protected:
+    // Helper for reading JSON files, protected so
+    // unit tests may access it via inheritance
+    std::string ReadJSON(VfsPath path);
+
 private:
     CSimulation2Impl* m;
 
-    // Helper for reading JSON files
-    std::string ReadJSON(VfsPath path);
-
     NONCOPYABLE(CSimulation2);
 };
 

source/simulation2/tests/test_ComponentManager.h

@@ -63 +63 @@
         man.LoadComponentTypes();
     }
 
+    void test_LoadScriptWithUTF8BOMSucceeds()
+    {
+        CSimContext context;
+        CComponentManager man(context, ScriptInterface::CreateRuntime());
+
+        bool isLoaded = man.LoadScript(L"simulation/components/test-bom-utf8.js");
+        TS_ASSERT(isLoaded);
+
+        // The script file loaded before adds a new property "Note" to the Engine
+        // object, as a simple proof it was executed at all.
+        ScriptInterface &scripts = man.GetScriptInterface();
+
+        CScriptVal engine;
+        bool hasEngine = scripts.GetProperty(scripts.GetGlobalObject(), "Engine", engine);
+
+        TS_ASSERT(hasEngine);
+
+        CScriptVal note;
+        bool hasNote = scripts.GetProperty(engine.get(), "Note", note);
+        TS_ASSERT(hasNote);
+
+        std::wstring noteText = scripts.ToString(note.get(), true);
+        TS_ASSERT_EQUALS(noteText, L"\"UTF8-BOM was accepted\"");
+    }
+
+    void test_LoadScriptWithoutBOMSucceeds()
+    {
+        CSimContext context;
+        CComponentManager man(context, ScriptInterface::CreateRuntime());
+
+        bool isLoaded = man.LoadScript(L"simulation/components/test-bom-none.js");
+        TS_ASSERT(isLoaded);
+
+        // The script file loaded before adds a new property "Note" to the Engine
+        // object, as a simple proof it was executed at all.
+        ScriptInterface &scripts = man.GetScriptInterface();
+
+        CScriptVal engine;
+        bool hasEngine = scripts.GetProperty(scripts.GetGlobalObject(), "Engine", engine);
+
+        TS_ASSERT(hasEngine);
+
+        CScriptVal note;
+        bool hasNote = scripts.GetProperty(engine.get(), "Note", note);
+        TS_ASSERT(hasNote);
+
+        std::wstring noteText = scripts.ToString(note.get(), true);
+
+        TS_ASSERT_EQUALS(noteText, L"\"Script without BOM was accepted\"");
+    }
+
     void test_LookupCID()
     {
         CSimContext context;

source/simulation2/tests/test_Simulation2.h

@@ -27 +27 @@
 #include "ps/Filesystem.h"
 #include "ps/XML/Xeromyces.h"
 
+/**
+ * Helper namespace to hold utility classes for testing of 'Simulation2'
+ * (prevent collision with similary-named stubs for other classes)
+ */
+namespace test_Simulation2
+{
+
+    /**
+     * Lightweight testing implementation of a virtual file system.
+     *
+     * While the TestSimulation2 class already has an associated mod (_test.sim),
+     * testing of the file loader routines via a stub has several advantages:
+     *  1.  Some routines, such as GetRMSData(), scan a whole directory whose path is
+     *      hard-coded in the testee. Using a real VFS might cause undesired interferences
+     *      as more tests are added for a single routine.
+     *  2.  A pure in-memory VFS is much faster.
+     *  3.  For UTF-8 tests, control characters (such as a leading BOM) may be expressed
+     *      directly in the C code, while they might be invisible in some editors when
+     *      put in real files.
+     */
+    struct StubVFS : public IVFS {
+
+        /**
+         * Simulates a file.
+         */
+        class StubFile
+        {
+        public:
+
+            /**
+             * Initializes a new stub file with fixed content.
+             * @param filename Name of the file.
+             * @param dataLength Size of the data held by the fake file.
+             * @param data Data found in the associated file.
+             */
+            StubFile(const char filename[], size_t dataLength, const u8 data[]) :
+                m_FileName(filename),
+                m_FileSize(dataLength)
+            {
+                u8 *dataCopy = new u8[dataLength];
+                memcpy(dataCopy, data, dataLength);
+                this->m_FileContent = shared_ptr<u8>(dataCopy);
+            }
+
+            /**
+             * Provides the data size of the file.
+             * @return An info record with file size and a fake date.
+             */
+            CFileInfo getInfo() const
+            {
+                return CFileInfo(this->getName(), this->m_FileSize, 0);
+            }
+
+            /**
+             * Retrieves the name of the file.
+             * @return The filename passed into the constructor.
+             */
+            OsPath getName() const
+            {
+                return OsPath(this->m_FileName);
+            }
+
+            /**
+             * Retrieves the data of this file.
+             * @return A copy of the data passed to the constructor.
+             */
+            shared_ptr<u8> getContent() const
+            {
+                return this->m_FileContent;
+            }
+
+        private:
+
+            /**
+             * Name of this file.
+             */
+            const char *m_FileName;
+
+            /**
+             * Data found in the file represented by this object
+             */
+            shared_ptr<u8> m_FileContent;
+
+            /**
+             * Data size of 'm_FileContent'
+             */
+            const size_t m_FileSize;
+        };
+
+        /**
+         * Creates a stub file system which simulates a fixed set of files.
+         * @param numberOfFiles Number of valid entries in 'files'.
+         * @param files The files which are provided (no real support for
+         *              directories, all files are returned on each call to
+         *              'GetDirectoryEntries').
+         */
+        StubVFS(size_t numberOfFiles, const StubFile files[]) :
+            m_NumberOfFiles(numberOfFiles)
+        {
+            this->m_Files = files;
+        }
+
+        virtual Status Mount(const VfsPath&, const OsPath&, size_t, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetFileInfo(const VfsPath& path, CFileInfo* info) const
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (this->m_Files[i].getName() == path.Filename())
+                {
+                    if (info)
+                    {
+                        *info = this->m_Files[i].getInfo();
+                    }
+                    return INFO::OK;
+                }
+            }
+            return CODE__PSRETURN_Scripting_LoadFile_OpenFailed;
+        }
+
+        virtual Status GetFilePriority(const VfsPath&, size_t*) const
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Retrieves a listing of all files held by this file system.
+         * @param infos Receives additional information about a file.
+         * @param names Receives the names of all files in the file system.
+         * @return OK.
+         */
+        virtual Status GetDirectoryEntries(const VfsPath&, CFileInfos* infos, DirectoryNames* names) const
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (infos)
+                {
+                    infos->push_back(this->m_Files[i].getInfo());
+                }
+                if (names)
+                {
+                    names->push_back(this->m_Files[i].getName());
+                }
+            }
+            return INFO::OK;
+        }
+
+        virtual Status CreateFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+
+            throw "Not yet implemented";
+        }
+
+        virtual Status ReplaceFile(const VfsPath&, const shared_ptr<u8>&, size_t)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * Read an entire file into memory.
+         *
+         * @param path Selects the fake file whose content is requested.
+         * @param fileContents receives a smart pointer to the contents.
+         *        CAVEAT: this will be taken from the file cache if the VFS was
+         *        created with cacheSize != 0 and size < cacheSize. There is no
+         *        provision for Copy-on-Write, which means that such buffers
+         *        must not be modified (this is enforced via mprotect).
+         * @param size receives the size [bytes] of the file contents.
+         * @return OK, if a fake file with given path was selected, or
+         *         OpenFailed in case the file does not exist.
+         **/
+        virtual Status LoadFile(const VfsPath& path, shared_ptr<u8>& fileContents, size_t& size)
+        {
+            for (size_t i = 0; i < this->m_NumberOfFiles; i++)
+            {
+                if (this->m_Files[i].getName() == path.Filename())
+                {
+                    fileContents = this->m_Files[i].getContent();
+                    size = this->m_Files[i].getInfo().Size();
+                    return INFO::OK;
+                }
+            }
+            return CODE__PSRETURN_Scripting_LoadFile_OpenFailed;
+        }
+
+        virtual std::wstring TextRepresentation() const
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetDirectoryRealPath(const VfsPath&, OsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status GetVirtualPath(const OsPath&, VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        /**
+         * remove file from the virtual directory listing and evict its
+         * data from the cache.
+         **/
+        virtual Status RemoveFile(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual Status RepopulateDirectory(const VfsPath&)
+        {
+            throw "Not yet implemented";
+        }
+
+        virtual void Clear()
+        {
+            throw "Not yet implemented";
+        }
+
+    private:
+
+        /**
+         * Number of valid elements in 'm_Files'.
+         */
+        const size_t m_NumberOfFiles;
+
+        /**
+         * The files found in this file system.
+         */
+        const StubFile *m_Files;
+    };
+
+}
+
 class TestSimulation2 : public CxxTest::TestSuite
 {
     void copyFile(const VfsPath& src, const VfsPath& dst)
@@ -39 +281 @@
 
     CTerrain m_Terrain;
 
+    /**
+     * Provides access to CSimulation2::ReadJSON() via inheritance
+     */
+    class CSimulation2Hook : public CSimulation2
+    {
+    public:
+        CSimulation2Hook(CUnitManager* unitManager, shared_ptr<ScriptRuntime> rt, CTerrain* terrain) :
+            CSimulation2(unitManager, rt, terrain)
+        {
+        }
+
+        std::string ReadJSON(VfsPath path)
+        {
+            return CSimulation2::ReadJSON(path);
+        }
+    };
+
+
 public:
     void setUp()
     {
@@ -127 +387 @@
         sim.BroadcastMessage(msg);
     }
 
+    /**
+     * Civ data files shall be loaded with utf8 byte-order-mark suppressed, if present.
+     */
+    void test_getcivdata_utf8support()
+    {
+        const u8 content1[] = "\xEF\xBB\xBF Asian characters";
+        const u8 content2[] = "}:-) wikings";
+        test_Simulation2::StubVFS::StubFile testFiles[] =
+        {   // Note: "- 1" suppresses the terminating \0 from the "file data"
+            test_Simulation2::StubVFS::StubFile("asian.json", sizeof(content1)/sizeof(content1[0]) - 1, content1),
+            test_Simulation2::StubVFS::StubFile("wikings.json", sizeof(content2)/sizeof(content2[0]) - 1, content2)
+        };
+
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_Simulation2::StubVFS(sizeof(testFiles)/sizeof(testFiles[0]), testFiles));
+
+        CSimulation2 sim(NULL, ScriptInterface::CreateRuntime(), &m_Terrain);
+
+        std::vector<std::string> civData = sim.GetCivData();
+
+        size_t numberOfCivs = civData.size();
+        TS_ASSERT_EQUALS(numberOfCivs, 2);
+
+        if (2 == numberOfCivs) // Prevent OutOfRange exception in case of failure
+        {
+            TS_ASSERT_EQUALS(civData[0], " Asian characters");
+            TS_ASSERT_EQUALS(civData[1], "}:-) wikings");
+        }
+
+        g_VFS = originalVFS;
+    }
+
+    /**
+     * Random map scripts/data shall be loaded with proper utf8 byte-order-mark
+     * support.
+     */
+    void test_getrmsdata_utf8support()
+    {
+        const u8 content1[] = "Maritime terrain <.))><";
+        const u8 content2[] = "\xEF\xBB\xBF Exotic language";
+        test_Simulation2::StubVFS::StubFile testFiles[] =
+        {   // Note: "- 1" suppresses the terminating \0 from the "file data"
+            test_Simulation2::StubVFS::StubFile("script1.json", sizeof(content1)/sizeof(content1[0]) - 1, content1),
+            test_Simulation2::StubVFS::StubFile("script2.json", sizeof(content2)/sizeof(content2[0]) - 1, content2)
+        };
+
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_Simulation2::StubVFS(sizeof(testFiles)/sizeof(testFiles[0]), testFiles));
+
+        CSimulation2 sim(NULL, ScriptInterface::CreateRuntime(), &m_Terrain);
+
+        std::vector<std::string> RMSScripts = sim.GetRMSData();
+
+        size_t numberOfScripts = RMSScripts.size();
+        TS_ASSERT_EQUALS(numberOfScripts, 2);
+
+        if (2 == numberOfScripts) // Prevent OutOfRange exception in case of failure
+        {
+            TS_ASSERT_EQUALS(RMSScripts[0], "Maritime terrain <.))><");
+            TS_ASSERT_EQUALS(RMSScripts[1], " Exotic language");
+        }
+
+        g_VFS = originalVFS;
+    }
+
     void test_hotload_scripts()
     {
         CSimulation2 sim(NULL, ScriptInterface::CreateRuntime(), &m_Terrain);
@@ -164 +489 @@
 
         TS_ASSERT_OK(sim.ReloadChangedFile(L"simulation/components/hotload/hotload.js"));
     }
+
+    void test_readjson_asciifilessupported()
+    {
+        const u8 content[] = "\xEF\xBB\xBF{id:'Raw data'}";
+        test_Simulation2::StubVFS::StubFile testFiles[] =
+        {   // Note: "- 1" suppresses the terminating \0 from the "file data"
+            test_Simulation2::StubVFS::StubFile("script.json", sizeof(content)/sizeof(content[0]) - 1, content)
+        };
+
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_Simulation2::StubVFS(sizeof(testFiles)/sizeof(testFiles[0]), testFiles));
+
+        CSimulation2Hook sim(NULL, ScriptInterface::CreateRuntime(), &m_Terrain);
+
+        std::string result = sim.ReadJSON("script.json");
+
+        TS_ASSERT_EQUALS(result, "{id:'Raw data'}");
+
+        g_VFS = originalVFS;
+    }
+
+    void test_readjson_utf8bomsupported()
+    {
+        const u8 content[] = "\xEF\xBB\xBF{id:'Blah'}";
+        test_Simulation2::StubVFS::StubFile testFiles[] =
+        {   // Note: "- 1" suppresses the terminating \0 from the "file data"
+            test_Simulation2::StubVFS::StubFile("script.json", sizeof(content)/sizeof(content[0]) - 1, content)
+        };
+
+        PIVFS originalVFS(g_VFS);
+        g_VFS = PIVFS(new test_Simulation2::StubVFS(sizeof(testFiles)/sizeof(testFiles[0]), testFiles));
+
+        CSimulation2Hook sim(NULL, ScriptInterface::CreateRuntime(), &m_Terrain);
+
+        std::string result = sim.ReadJSON("script.json");
+
+        TS_ASSERT_EQUALS(result, "{id:'Blah'}");
+
+        g_VFS = originalVFS;
+    }
+
 };