diff --git a/src/font.cpp b/src/font.cpp
index 78668354243..67e5f3420d0 100644
--- a/src/font.cpp
+++ b/src/font.cpp
@@ -32,6 +32,7 @@
 #	include FT_FREETYPE_H
 #	include FT_BITMAP_H
 #	include FT_MODULE_H
+#   include FT_TRUETYPE_TABLES_H
 #endif
 
 #ifdef HAVE_HARFBUZZ
@@ -145,6 +146,8 @@ namespace {
 		void vApplyStyle(const Style& style) override;
 
 	private:
+		void SetSize(int height, bool create);
+
 		FT_Face face = nullptr;
 		std::vector<uint8_t> ft_buffer;
 		// Freetype uses the baseline as 0 and the built-in fonts the top
@@ -287,20 +290,7 @@ FTFont::FTFont(Filesystem_Stream::InputStream is, int size, bool bold, bool ital
 		}
 	}
 
-	if (FT_HAS_COLOR(face)) {
-		// FIXME: Find the best size
-		FT_Select_Size(face, 0);
-	} else {
-		FT_Set_Pixel_Sizes(face, 0, size);
-	}
-
-#ifdef HAVE_HARFBUZZ
-	hb_buffer = hb_buffer_create();
-	hb_font = hb_ft_font_create_referenced(face);
-	hb_ft_font_set_funcs(hb_font);
-#endif
-
-	baseline_offset = static_cast<int>(size * (10.0 / 12.0));
+	SetSize(size, true);
 
 	if (!strcmp(face->family_name, "RM2000") || !strcmp(face->family_name, "RMG2000")) {
 		// Workaround for bad kerning in RM2000 and RMG2000 fonts
@@ -501,21 +491,56 @@ void FTFont::vApplyStyle(const Style& style) {
 		return;
 	}
 
+	SetSize(style.size, false);
+}
+
+void FTFont::SetSize(int height, bool create) {
 	if (FT_HAS_COLOR(face)) {
 		// FIXME: Find the best size
 		FT_Select_Size(face, 0);
+	} else if (FT_IS_SCALABLE(face)) {
+		// Calculate the pt size from px
+		auto table_os2 = static_cast<TT_OS2*>(FT_Get_Sfnt_Table(face, ft_sfnt_os2));
+		auto table_hori = static_cast<TT_HoriHeader*>(FT_Get_Sfnt_Table(face, ft_sfnt_hhea));
+
+		if (table_os2 && table_hori) {
+			int units;
+			if (table_os2->usWinAscent + table_os2->usWinDescent == 0) {
+				units = table_hori->Ascender - table_hori->Descender;
+			} else {
+				units = table_os2->usWinAscent + table_os2->usWinDescent;
+			}
+
+			int pt = FT_MulDiv(face->units_per_EM, height, units);
+			if (FT_MulDiv(units, pt, face->units_per_EM) > height) {
+				--pt;
+			}
+
+			height = std::max<int>(1, pt);
+		}
+
+		FT_Set_Pixel_Sizes(face, 0, height);
 	} else {
-		FT_Set_Pixel_Sizes(face, 0, style.size);
+		FT_Set_Pixel_Sizes(face, 0, face->available_sizes->height);
 	}
 
 #ifdef HAVE_HARFBUZZ
-	// Without this the sizes become desynchronized
-	hb_font_destroy(hb_font);
+	if (create) {
+		hb_buffer = hb_buffer_create();
+	} else {
+		// Without this the sizes become desynchronized
+		hb_font_destroy(hb_font);
+	}
 	hb_font = hb_ft_font_create_referenced(face);
 	hb_ft_font_set_funcs(hb_font);
 #endif
-}
 
+	baseline_offset = static_cast<int>(FT_MulFix(face->ascender, face->size->metrics.y_scale) / 64);
+	if (baseline_offset == 0) {
+		// FIXME: Becomes 0 for FON files. How is the baseline calculated for them?
+		baseline_offset = static_cast<int>(height * (10.0 / 12.0));
+	}
+}
 #endif
 
 FontRef Font::Default() {
@@ -567,7 +592,7 @@ FontRef Font::CreateFtFont(Filesystem_Stream::InputStream is, int size, bool bol
 
 	FreeFontMemory();
 
-	std::string key = ToString(is.GetName()) + ":" + (bold ? "T" : "F") + (italic ? "T" : "F");
+	std::string key = ToString(is.GetName()) + ":" + std::to_string(size) + ":" + (bold ? "T" : "F") + (italic ? "T" : "F");
 
 	auto it = ft_cache.find(key);
 
diff --git a/src/game_interpreter.cpp b/src/game_interpreter.cpp
index a12785e08e3..acb52274406 100644
--- a/src/game_interpreter.cpp
+++ b/src/game_interpreter.cpp
@@ -4324,14 +4324,6 @@ bool Game_Interpreter::CommandManiacShowStringPicture(lcf::rpg::EventCommand con
 	text.line_spacing = (flags & (0xFF << 16)) >> 16;
 	text.font_size = ValueOrVariableBitfield(com.parameters[0], 5, com.parameters[20]);
 
-	// Windows uses pt but we use px
-	int font_px = Utils::RoundTo<int>(text.font_size * (72 / 96.0));
-	// Maniac Patch appears to confuse pt and px causing large padding everywhere
-	// The next two lines emulate this problem
-	text.position_y = text.font_size - font_px;
-	text.line_spacing += text.font_size - font_px;
-
-	text.font_size = font_px;
 
 	// maniacs stores string parsing modes in the delimiters
 	// x01 -> literal string
diff --git a/src/game_windows.cpp b/src/game_windows.cpp
index 4ca722385ae..de678dc2170 100644
--- a/src/game_windows.cpp
+++ b/src/game_windows.cpp
@@ -380,7 +380,7 @@ void Game_Windows::Window_User::Refresh(bool& async_wait) {
 		auto style_guard = apply_style(font, text);
 
 		int x = text.position_x;
-		int y = text.position_y;
+		int y = text.position_y + 2; // +2 to match the offset RPG_RT uses
 		int text_color = 0;
 		for (const auto& line: pm.GetLines()) {
 			std::u32string line32;