Added exception checking for invalid bitmap formats, updated README t…

…o help with Memory address debugging

.gitignore

@@ -2,6 +2,9 @@
 *__pycache__*
 MAMEToolkit/emulator/mame/roms/sfiii3n.zip
 MAMEToolkit/emulator/mame/mame
+MAMEToolkit/emulator/mame/nvram
+MAMEToolkit/emulator/mame/cfg
+build
 dist
 MAMEToolkit-**
 MAMEToolkit.egg-info

MAMEToolkit/emulator/BitmapFormat.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class BitmapFormat(Enum):
+    RGB32 = 3
+    ARGB32 = 4

MAMEToolkit/emulator/Console.py

@@ -14,12 +14,16 @@ class Console(object):
     # render is for displaying the frames to the emulator window, disabling it has little to no effect
     # throttle enabled will run any game at the intended gameplay speed, disabling it will run the game as fast as the computer can handle
     # debug enabled will print everything that comes out of the Lua engine console
-    def __init__(self, roms_path, game_id, render=True, throttle=False, debug=False):
+    def __init__(self, roms_path, game_id, cheat_debugger=False, render=True, throttle=False, debug=False):
         self.logger = logging.getLogger("Console")
 
         command = f"exec ./mame -rompath '{str(Path(roms_path).absolute())}' -pluginspath plugins -skip_gameinfo -sound none -console "+game_id
         if not render:
             command += " -video none"
+
+        if cheat_debugger:
+            command += " -debug"
+
         if throttle:
             command += " -throttle"
         else:

MAMEToolkit/emulator/Emulator.py

@@ -3,6 +3,7 @@
 from MAMEToolkit.emulator.Console import Console
 from MAMEToolkit.emulator.pipes.Pipe import Pipe
 from MAMEToolkit.emulator.pipes.DataPipe import DataPipe
+from MAMEToolkit.emulator.BitmapFormat import BitmapFormat
 
 
 # Converts a list of action Enums into the relevant Lua engine representation
@@ -24,6 +25,14 @@ def list_actions(roms_path, game_id):
     return actions
 
 
+def see_games():
+    Emulator("env1", "", "", {})
+
+
+def run_cheat_debugger(roms_path, game_id):
+    Console(roms_path, game_id, cheat_debugger=True, render=True, throttle=True, debug=True)
+
+
 # An interface for using the Lua engine console functionality
 class Emulator(object):
 
@@ -41,6 +50,7 @@ def __init__(self, env_id, roms_path, game_id, memory_addresses, frame_ratio=3,
         atexit.register(self.close)
         self.wait_for_resource_registration()
         self.create_lua_variables()
+        bitmap_format = self.get_bitmap_format()
         screen_width = self.setup_screen_width()
         screen_height = self.setup_screen_height()
         self.screenDims = {"width": screen_width, "height": screen_height}
@@ -50,12 +60,25 @@ def __init__(self, env_id, roms_path, game_id, memory_addresses, frame_ratio=3,
         self.actionPipe = Pipe(env_id, "action", 'w', pipes_path)
         self.actionPipe.open(self.console)
 
-        self.dataPipe = DataPipe(env_id, self.screenDims, memory_addresses, pipes_path)
+        self.dataPipe = DataPipe(env_id, self.screenDims, bitmap_format, memory_addresses, pipes_path)
         self.dataPipe.open(self.console)
 
         # Connect inter process communication
         self.setup_frame_access_loop()
 
+    def get_bitmap_format(self):
+        bitmap_format = self.console.writeln('print(s:bitmap_format())', expect_output=True)
+        if len(bitmap_format) != 1:
+            raise IOError('Expected one result from "print(s:bitmap_format())", but received: ', bitmap_format)
+        try:
+            return {
+                "RGB32 - 32bpp 8-8-8 RGB": BitmapFormat.RGB32,
+                "ARGB32 - 32bpp 8-8-8-8 ARGB": BitmapFormat.ARGB32
+            }[bitmap_format[0]]
+        except KeyError:
+            self.console.close()
+            raise EnvironmentError("MAMEToolkit only supports RGB32 and ARGB32 frame bit format games")
+
     def create_lua_variables(self):
         self.console.writeln('iop = manager:machine():ioport()')
         self.console.writeln('s = manager:machine().screens[":screen"]')

MAMEToolkit/emulator/__init__.py

@@ -1,3 +1,3 @@
 from MAMEToolkit.emulator.Action import Action
-from MAMEToolkit.emulator.Emulator import Emulator, list_actions
+from MAMEToolkit.emulator.Emulator import Emulator, list_actions, see_games, run_cheat_debugger
 from MAMEToolkit.emulator.Address import Address

MAMEToolkit/emulator/pipes/DataPipe.py

@@ -1,13 +1,15 @@
 import numpy as np
 from MAMEToolkit.emulator.pipes.Pipe import Pipe
+from MAMEToolkit.emulator.BitmapFormat import BitmapFormat
 
 
 # A special implementation of a Linux FIFO pipe which is used for reading all of the frame data and memory address values from the emulator
 class DataPipe(object):
 
-    def __init__(self, env_id, screen_dims, addresses, pipes_path):
+    def __init__(self, env_id, screen_dims, bitmap_format: BitmapFormat, addresses, pipes_path):
         self.pipe = Pipe(env_id, "data", 'r', pipes_path)
         self.screenDims = screen_dims
+        self.bitmap_format = bitmap_format
         self.addresses = addresses
 
     def open(self, console):
@@ -31,6 +33,6 @@ def read_data(self, timeout=10):
             part = line[cursor:cursor_end]
             data[k] = int(part.decode("utf-8"))
             cursor = cursor_end+1
-        data["frame"] = np.frombuffer(line[cursor:], dtype='uint8').reshape(self.screenDims["height"], self.screenDims["width"], 3)
+        data["frame"] = np.frombuffer(line[cursor:], dtype='uint8').reshape(self.screenDims["height"], self.screenDims["width"], self.bitmap_format.value)
         return data
 

README.md

@@ -11,13 +11,15 @@ pip install MAMEToolkit
 
 **DISCLAIMER: We are unable to provide you with any game ROMs. It is the users own legal responsibility to acquire a game ROM for emulation. This library should only be used for non-commercial research purposes.**
 
+There are some free ROMs available at: [https://www.mamedev.org/roms/]
+
 ## Street Fighter Random Agent Demo
 The toolkit has currently been applied to Street Fighter III Third Strike: Fight for the Future (Japan 990608, NO CD), but can modified for any game available on MAME. The following demonstrates how a random agent can be written for a street fighter environment.
 ```python
 import random
 from MAMEToolkit.sf_environment import Environment
 
-roms_path = "roms/"
+roms_path = "roms/"  # Replace this with the path to your ROMs
 env = Environment("env1", roms_path)
 env.start()
 while True:
@@ -56,7 +58,7 @@ def run_env(env):
 def main():
     workers = 8
     # Environments must be created outside of the threads
-    roms_path = "roms/"
+    roms_path = "roms/"  # Replace this with the path to your ROMs
     envs = [Environment(f"env{i}", roms_path) for i in range(workers)]
     threads = [Thread(target=run_env, args=(envs[i], )) for i in range(workers)]
     [thread.start() for thread in threads]
@@ -65,25 +67,37 @@ def main():
 ![](pics/hogwild3.gif "Hogwild Random Agents")
 
 ## Setting Up Your Own Game Environment
-It doesn't take much to interact with the emulator itself using the toolkit, however the challenge comes from finding the memory address values associated with the internal state you care about, and tracking said state with your environment class.
-The internal memory states of a game can be tracked using the [MAME Cheat Debugger](http://docs.mamedev.org/debugger/cheats.html), which allows you to track how the memory address values of the game change over time.
-To create an emulation of the game you must first have the ROM for the game you are emulating and know the game ID used by MAME, for example for this version of street fighter it is 'sfiii3n'. 
 
 **Game ID's**<br>
+To create an emulation of the game you must first have the ROM for the game you are emulating and know the game ID used by MAME, for example for this version of street fighter it is 'sfiii3n'. 
 The id of your game can be found by running:
 ```python
-from MAMEToolkit.emulator import Emulator
-emulator = Emulator("env1", "", "", memory_addresses)
+from MAMEToolkit.emulator import see_games
+see_games()
 ```
 This will bring up the MAME emulator. You can search through the list of games to find the one you want. The id of the game is always in brackets at the end of the game title.
 
 **Memory Addresses**<br>
-Once you have these and have determined the memory addresses you wish to track you can start the emulation:
+It doesn't take much to interact with the emulator itself using the toolkit, however the challenge comes from finding the memory address values associated with the internal state you care about, and tracking said state with your environment class.
+The internal memory states of a game can be tracked using the [MAME Cheat Debugger](http://docs.mamedev.org/debugger/cheats.html), which allows you to track how the memory address values of the game change over time.
+
+
+The cheat debugger can be run using the following:
+```python
+from MAMEToolkit.emulator import run_cheat_debugger
+roms_path = "roms/" # Replace this with the path to your ROMs
+game_id = "sfiii3n"
+run_cheat_debugger(roms_path, game_id)
+```
+For information about using the debugger, see the Memory dump section of this tutorial [https://www.dorkbotpdx.org/blog/skinny/use_mames_debugger_to_reverse_engineer_and_extend_old_games]
+
+
+Once you have determined the memory addresses you wish to track you can start the emulation using:
 ```python
 from MAMEToolkit.emulator import Emulator
 from MAMEToolkit.emulator import Address
 
-roms_path = "roms/"
+roms_path = "roms/"  # Replace this with the path to your ROMs
 game_id = "sfiii3n"
 memory_addresses = {
         "fighting": Address('0x0200EE44', 'u8'),
@@ -95,7 +109,7 @@ memory_addresses = {
 
 emulator = Emulator("env1", roms_path, "sfiii3n", memory_addresses)
 ```
-This will immediately start the emulation and halt it when it toolkit has linked to the emulator process. 
+This will immediately start the emulation and halt it when the toolkit has linked to the emulator process. 
 
 **Stepping the emulator**<br>
 Once the toolkit is linked, you can step the emulator along using the step function:
@@ -123,7 +137,7 @@ To identify which ports are availble use the list actions command:
 ```python
 from MAMEToolkit.emulator import list_actions
 
-roms_path = "roms/"
+roms_path = "roms/"  # Replace this with the path to your ROMs
 game_id = "sfiii3n"
 print(list_actions(roms_path, game_id))
 ```
@@ -179,4 +193,22 @@ To ensure that the toolkit is able to train algorithms, a simple 5 layer ConvNet
 
 ![](pics/chart.png "ConvNet Results")
 
+## MAME Changes
+The library works by acting as a wrapper around a modified MAME implementation.
+The following changes were made:
+* Updated the lua console to allow for the retrieval of the format of frame data
+* Update the lua console to allow for the retrieval of the current frames data
+* Disabled game start warnings
+
+The following files are affected:
+* src/emu/machine.cpp
+* src/emu/video.cpp
+* src/emu/video.h
+* src/frontend/mame/luaengine.cpp
+* src/frontend/mame/ui/ui.cpp
+* src/osd/sdl/window.cpp
+
+**The modified MAME implementation can be found at [https://github.com/M-J-Murray/mame]**
+
+
 

test/emulator/EmulatorTest.py

@@ -3,10 +3,11 @@
 
 from MAMEToolkit.emulator.Emulator import Emulator
 from MAMEToolkit.emulator.Address import Address
-from multiprocessing import set_start_method, Process, Queue
+from multiprocessing import Process, Queue
+from time import sleep
 
 
-def run_emulator(env_id, game_id, roms_path, memory_addresses, output_queue):
+def run_emulator(env_id, roms_path, game_id, memory_addresses, output_queue):
     emulator = None
     try:
         emulator = Emulator(env_id, roms_path, game_id, memory_addresses)
@@ -20,9 +21,10 @@ class EmulatorTest(unittest.TestCase):
     def test_screen_dimensions(self):
         memory_addresses = {"test": Address("02000008", "u8")}
         game_id = "sfiii3n"
+        roms_path = "/home/michael/dev/MAMEToolkit/MAMEToolkit/emulator/mame/roms"
         emulator = None
         try:
-            emulator = Emulator("testEnv1", "/home/michael/dev/MAMEToolkit/MAMEToolkit/emulator/mame/roms", game_id, memory_addresses)
+            emulator = Emulator("testEnv1", roms_path, game_id, memory_addresses)
             assert_that(emulator.screenDims["width"], equal_to(384))
             assert_that(emulator.screenDims["height"], equal_to(224))
         finally:
@@ -31,24 +33,26 @@ def test_screen_dimensions(self):
     def test_step(self):
         memory_addresses = {"test": Address("02000008", "u8")}
         game_id = "sfiii3n"
+        roms_path = "/home/michael/dev/MAMEToolkit/MAMEToolkit/emulator/mame/roms"
         emulator = None
         try:
-            emulator = Emulator("testEnv1", game_id, memory_addresses)
+            emulator = Emulator("testEnv1", roms_path, game_id, memory_addresses)
             data = emulator.step([])
             assert_that(data["frame"].shape, equal_to((224, 384, 3)))
             assert_that(data["test"], equal_to(0))
         finally:
             emulator.close()
 
     def test_multiprocessing(self):
-        set_start_method("spawn")
-        workers = 10
+        workers = 2
         game_id = "sfiii3n"
+        roms_path = "/home/michael/dev/MAMEToolkit/MAMEToolkit/emulator/mame/roms"
         memory_addresses = {"test": Address("02000008", "u8")}
         output_queue = Queue()
-        processes = [Process(target=run_emulator, args=[f"testEnv{i}", "/home/michael/dev/MAMEToolkit/MAMEToolkit/emulator/mame/roms", game_id, memory_addresses, output_queue]) for i in range(workers)]
+        processes = [Process(target=run_emulator, args=[f"testEnv{i}", roms_path, game_id, memory_addresses, output_queue]) for i in range(workers)]
         [process.start() for process in processes]
-        [process.join() for process in processes]
+        sleep(14)
+        [process.join(timeout=1) for process in processes]
         for i in range(workers):
             data = output_queue.get(timeout=0.1)
             assert_that(data["frame"].shape, equal_to((224, 384, 3)))