--- np2/i386c/ia32/paging.c	2004/01/13 16:37:42	1.3
+++ np2/i386c/ia32/paging.c	2012/01/23 06:01:44	1.38
@@ -1,7 +1,5 @@
-/*	$Id: paging.c,v 1.3 2004/01/13 16:37:42 monaka Exp $	*/
-
 /*
- * Copyright (c) 2003 NONAKA Kimihiro
+ * Copyright (c) 2003-2004 NONAKA Kimihiro
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -12,8 +10,6 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
@@ -32,112 +28,25 @@
 #include "ia32.mcr"
 
 /*
- * 矢□斥’犯奴伊弁玄伉’巨件玄伉 (4K 田奶玄矢□斥銀迕凜)
- *
- *  31                                    12 11   9 8  7 6 5  4   3   2   1  0 
- * +----------------------------------------+------+-+--+-+-+---+---+---+---+-+
- * |   矢□斥’氾□皮伙及矛□旦’失玉伊旦   |銀迕第|G|PS|0|A|PCD|PWT|U/S|R/W|P|
- * +----------------------------------------+------+-+--+-+-+---+---+---+---+-+
- *                                              |   |  | | |  |   |   |   |  |
- * 9-11: 扑旦氾丞’皿伕弘仿穴互銀迕第チ --------+   |  | | |  |   |   |   |  |
- *    8: 弘伕□田伙’矢□斥(拑骰今木月) ------------+  | | |  |   |   |   |  |
- *    7: 矢□斥’扔奶朮 (0 = 4k 田奶玄矢□斥) ---------+ | |  |   |   |   |  |
- *    6: 芺沶 (0) ---------------------------------------+ |  |   |   |   |  |
- *    5: 失弁本旦 -----------------------------------------+  |   |   |   |  |
- *    4: 平乓永扑亙拑躲 --------------------------------------+   |   |   |  |
- *    3: 仿奶玄旦伙□ --------------------------------------------+   |   |  |
- *    2: 交□扒▼旦□由田奶扒 (0 = 旦□由田奶扒) ---------------------+   |  |
- *    1: 厶心潸曰▼踏五慇心 (0 = 厶心潸曰及心) ---------------------------+  |
- *    0: 矢□斥繡箕 ---------------------------------------------------------+
- */
-#define	CPU_PDE_BASEADDR_MASK	0xfffff000
-#define	CPU_PDE_PAGE_SIZE	(1 << 7)
-#define	CPU_PDE_ACCESS		(1 << 5)
-#define	CPU_PDE_CACHE_DISABLE	(1 << 4)
-#define	CPU_PDE_WRITE_THROUGH	(1 << 3)
-#define	CPU_PDE_USER_MODE	(1 << 2)
-#define	CPU_PDE_WRITABLE	(1 << 1)
-#define	CPU_PDE_PRESENT		(1 << 0)
-
-/*
- * 矢□斥’犯奴伊弁玄伉’巨件玄伉 (4M 田奶玄矢□斥銀迕凜)
- * 
- *  31                        22 21       12 11   9 8  7 6 5  4   3   2   1  0 
- * +----------------------------+-----------+------+-+--+-+-+---+---+---+---+-+
- * |矢□斥氾□皮伙及坁咥失玉伊旦|  芺沶碧心 |銀迕第|G|PS|D|A|PCD|PWT|U/S|R/W|P|
- * +----------------------------+-----------+------+-+--+-+-+---+---+---+---+-+
- *                                              |   |  | | |  |   |   |   |  |
- * 9-11: 扑旦氾丞’皿伕弘仿穴互銀迕第チ --------+   |  | | |  |   |   |   |  |
- *    8: 弘伕□田伙’矢□斥 ------------------------+  | | |  |   |   |   |  |
- *    7: 矢□斥’扔奶朮 (1 = 4M 田奶玄矢□斥) ---------+ | |  |   |   |   |  |
- *    6: 母□氾奴 ---------------------------------------+ |  |   |   |   |  |
- *    5: 失弁本旦 -----------------------------------------+  |   |   |   |  |
- *    4: 平乓永扑亙拑躲 --------------------------------------+   |   |   |  |
- *    3: 仿奶玄旦伙□ --------------------------------------------+   |   |  |
- *    2: 交□扒▼旦□由田奶扒 (0 = 旦□由田奶扒) ---------------------+   |  |
- *    1: 厶心潸曰▼踏五慇心 (0 = 厶心潸曰及心) ---------------------------+  |
- *    0: 矢□斥繡箕 ---------------------------------------------------------+
- */
-#define	CPU_PDE_4M_BASEADDR_MASK	0xffc00000
-#define	CPU_PDE_4M_GLOBAL_PAGE		(1 << 8)
-#define	CPU_PDE_4M_PAGE_SIZE		(1 << 7)
-#define	CPU_PDE_4M_DIRTY		(1 << 6)
-#define	CPU_PDE_4M_ACCESS		(1 << 5)
-#define	CPU_PDE_4M_CACHE_DISABLE	(1 << 4)
-#define	CPU_PDE_4M_WRITE_THROUGH	(1 << 3)
-#define	CPU_PDE_4M_USER_MODE		(1 << 2)
-#define	CPU_PDE_4M_WRITABLE		(1 << 1)
-#define	CPU_PDE_4M_PRESENT		(1 << 0)
-
-/*
- * 矢□斥’氾□皮伙’巨件玄伉 (4k 田奶玄’矢□斥)
+ * 矢□斥白巧伙玄拺陸
  *
- *  31                                    12 11   9 8 7 6 5  4   3   2   1  0 
- * +----------------------------------------+------+-+-+-+-+---+---+---+---+-+
- * |        矢□斥及矛□旦’失玉伊旦        |銀迕第|G|0|D|A|PCD|PWT|U/S|R/W|P|
- * +----------------------------------------+------+-+-+-+-+---+---+---+---+-+
- *                                              |   | | | |  |   |   |   |  |
- *  9-11: 扑旦氾丞’皿伕弘仿穴互銀迕第チ -------+   | | | |  |   |   |   |  |
- *     8: 弘伕□田伙’矢□斥 -----------------------+ | | |  |   |   |   |  |
- *     7: 芺沶 (0) -----------------------------------+ | |  |   |   |   |  |
- *     6: 母□氾奴 -------------------------------------+ |  |   |   |   |  |
- *     5: 失弁本旦 ---------------------------------------+  |   |   |   |  |
- *     4: 平乓永扑亙拑躲 ------------------------------------+   |   |   |  |
- *     3: 仿奶玄旦伙□ ------------------------------------------+   |   |  |
- *     2: 交□扒▼旦□由田奶扒 (0 = 旦□由田奶扒) -------------------+   |  |
- *     1: 厶心潸曰▼踏五慇心 (0 = 厶心潸曰及心) -------------------------+  |
- *     0: 矢□斥繡箕 -------------------------------------------------------+
+ * 4-31: 鈭碧心
+ *    3: RSVD: 0 = 白巧伙玄及葆奢反芺沶申永玄啜�匹反卅井勻凶‘
+ *             1 = 矢□斥’白巧伙玄及葆奢反﹞啜�午穴□弁今木凶 PTE 引凶反
+ *                 PDE 及芺沶申永玄匏譆及丹切域勾匹﹞1 互腹請今木凶仇午匹丐月‘
+ *    2: U/S:  0 = 白巧伙玄及葆奢午卅勻凶失弁本旦反皿伕本永扔互旦□由田奶扒’
+ *                 乒□玉匹撢墊醱卞墊歹木凶‘
+ *             1 = 白巧伙玄及葆奢午卅勻凶失弁本旦反皿伕本永扔互交□扒’乒□玉匹
+ *                 撢墊醱卞墊歹木凶‘
+ *    1: W/R:  0 = 白巧伙玄及葆奢午卅勻凶失弁本旦互厶心潸曰匹丐勻凶‘
+ *             1 = 白巧伙玄及葆奢午卅勻凶失弁本旦互踏五慇心匹丐勻凶‘
+ *    0: P:    0 = 白巧伙玄及葆奢互尕箕矢□斥匹丐勻凶‘
+ *             1 = 白巧伙玄及葆奢互矢□斥’伊矛伙忡賃啜�匹丐勻凶‘
  */
-#define	CPU_PTE_BASEADDR_MASK	0xfffff000
-#define	CPU_PTE_GLOBAL_PAGE	(1 << 8)
-#define	CPU_PTE_DIRTY		(1 << 6)
-#define	CPU_PTE_ACCESS		(1 << 5)
-#define	CPU_PTE_CACHE_DISABLE	(1 << 4)
-#define	CPU_PTE_WRITE_THROUGH	(1 << 3)
-#define	CPU_PTE_USER_MODE	(1 << 2)
-#define	CPU_PTE_WRITABLE	(1 << 1)
-#define	CPU_PTE_PRESENT		(1 << 0)
 
 /*
- * 矢□斥白巧伙玄拺陸
- *
- * 4-31: 芺沶碧心
- *    3: RSVD: 0 = 白巧伙玄及葆奢反芺沶申永玄啜�匹反卅井勻凶‘
- *             1 = 矢□斥’白巧伙玄及葆奢反﹞啜�午穴□弁今木凶 PTE 引凶反
- *                 PDE 及芺沶申永玄匏譆及丹切域勾匹﹞1 互腹請今木凶仇午匹丐月‘
- *    2: U/S:  0 = 白巧伙玄及葆奢午卅勻凶失弁本旦反皿伕本永扔互旦□由田奶扒’
- *                 乒□玉匹撢墊醱卞墊歹木凶‘
- *             1 = 白巧伙玄及葆奢午卅勻凶失弁本旦反皿伕本永扔互交□扒’乒□玉匹
- *                 撢墊醱卞墊歹木凶‘
- *    1: W/R:  0 = 白巧伙玄及葆奢午卅勻凶失弁本旦互厶心潸曰匹丐勻凶‘
- *             1 = 白巧伙玄及葆奢午卅勻凶失弁本旦互踏五慇心匹丐勻凶‘
- *    0: P:    0 = 白巧伙玄及葆奢互尕箕矢□斥匹丐勻凶‘
- *             1 = 白巧伙玄及葆奢互矢□斥’伊矛伙忡賃啜�匹丐勻凶‘
- */
-
-/*
- * 票摒 4.12. 矢□斥忡賃午本弘丟件玄忡賃及賺心寧歹六
- * ＞刓 4-2. 矢□斥’犯奴伊弁玄伉午矢□斥’氾□皮伙及忡賃及賺心寧歹六＝
+ * 銝 4.12. 矢□斥忡賃午本弘丟件玄忡賃及賺心寧歹六
+ * ＞刓 4-2. 矢□斥’犯奴伊弁玄伉午矢□斥’氾□皮伙及忡賃及賺心寧歹六＝
  *
  * +------------+------------+------------+
  * |    PDE     |    PTE     |   merge    |
@@ -162,11 +71,11 @@
  * |  s  |  rw  |  s  |  rw  |  s  |  rw  |
  * +-----+------+-----+------+-----+------+
  *
- * ＞奴 rw/p : CR0 ﹜石 WP ’乒’氾’生﹜乓 ON ﹜石本艭謢甲﹜穴 ro
+ * 蜬扔 rw/p : CR0 槦亦 WP 樍］乓樍啣∫ ON 槦亦磝巨潞啣╲槦永 ro
  */
 
 /*
- * ’漰銣籈＞’永’扔’弗/PxE(本篞亙扔奶本生)/CPL/CR0 ﹜生’伊﹝扑’弁’＞’永’扔’弗弁＞﹜石巨伉平弁
+ * 樍﹝樍＞樍尼槻＞槻永槻扔槻弗/PxE(頨湊奴剚賤翴央)/CPL/CR0 槦奴樍婍汙槻弁槻＞槻永槻扔槻弗鴘孕槦亦壚＞餑   *
  * +-----+-----+-----+-----+-----+---+
  * | CR0 | CPL | PxE | PxE | ope |   |
@@ -199,9 +108,9 @@
  * +-----+-----------+-----+-----+---+
  */
 #if !defined(USE_PAGE_ACCESS_TABLE)
-static const DWORD page_access = 0xd0cdd0ff;
+#define	page_access	0xd0ddd0ff
 #else	/* USE_PAGE_ACCESS_TABLE */
-static const BYTE page_access_bit[32] = {
+static const UINT8 page_access_bit[32] = {
 	1,	/* CR0: n, CPL: s, PTE: s, PTE: r, ope: r */
 	1,	/* CR0: n, CPL: s, PTE: s, PTE: r, ope: w */
 	1,	/* CR0: n, CPL: s, PTE: s, PTE: w, ope: r */
@@ -230,7 +139,7 @@ static const BYTE page_access_bit[32] = 
 	1,	/* CR0: p, CPL: s, PTE: u, PTE: r, ope: r */
 	0,	/* CR0: p, CPL: s, PTE: u, PTE: r, ope: w */
 	1,	/* CR0: p, CPL: s, PTE: u, PTE: w, ope: r */
-	0,	/* CR0: p, CPL: s, PTE: u, PTE: w, ope: w */
+	1,	/* CR0: p, CPL: s, PTE: u, PTE: w, ope: w */
 
 	0,	/* CR0: p, CPL: u, PTE: s, PTE: r, ope: r */
 	0,	/* CR0: p, CPL: u, PTE: s, PTE: r, ope: w */
@@ -246,17 +155,17 @@ static const BYTE page_access_bit[32] = 
 
 /*
  *--
- * 32bit 坁咥失玉伊旦 4k 矢□斥
+ * 32bit 臄孕艛砍中樍脣旨槻弗 4k 樍婍汙槻弁
  *
- * ’籈甲’＞﹝仲’＞’用’孎弗
+ * 樍尼樍嗅中樍扔槻＞樍脣旨槻弗
  *  31                    22 21                  12 11                       0
  * +------------------------+----------------------+--------------------------+
- * |  ’伊﹝扑’弁﹝仲’甘’＝’孎永’生’ |   矢□斥’氾□皮伙   |        左白本永玄        |
+ * |  樍婍汙槻弁樍扔樍桀丰樍乓槻永樍啣收  |   樍婍汙槻弁樍扔樍砍汙樍极早   |        槻尼樍忴公樍血亥        |
  * +------------------------+----------------------+--------------------------+
  *             |                        |                       |
  * +-----------+            +-----------+                       +----------+
  * |                        |                                              |
- * |  ’伊﹝扑’弁﹝仲’甘’＝’孎永’生’ |   矢□斥’氾□皮伙            矢□斥         |
+ * |  矢□斥’犯奴伊弁玄伉  |   矢□斥’氾□皮伙            矢□斥         |
  * | +--------------------+ | +-------------------+   +------------------+ |
  * | |                    | | |                   |   |                  | |
  * | |                    | | +-------------------+   |                  | |
@@ -268,160 +177,723 @@ static const BYTE page_access_bit[32] = 
  *   |                    | | |                   | | |                  |
  * +>+--------------------+ +>+-------------------+ +>+------------------+
  * |
- * +- CR3()
+ * +- CR3(坁咥失玉伊旦)
  */
 
-static DWORD paging(DWORD laddr, int crw, int user_mode);
-#if defined(IA32_SUPPORT_TLB)
-static BOOL tlb_lookup(DWORD vaddr, int crw, DWORD* paddr);
-static void tlb_update(DWORD paddr, DWORD entry, int crw);
-#endif
+/* TLB */
+struct tlb_entry {
+	UINT32	tag;	/* linear address */
+#define	TLB_ENTRY_TAG_VALID		(1 << 0)
+/*	pde & pte & CPU_PTE_WRITABLE	(1 << 1)	*/
+/*	pde & pte & CPU_PTE_USER_MODE	(1 << 2)	*/
+#define	TLB_ENTRY_TAG_DIRTY		CPU_PTE_DIRTY		/* (1 << 6) */
+#define	TLB_ENTRY_TAG_GLOBAL		CPU_PTE_GLOBAL_PAGE	/* (1 << 8) */
+#define	TLB_ENTRY_TAG_MAX_SHIFT		12
+	UINT32	paddr;	/* physical address */
+};
+static void MEMCALL tlb_update(UINT32 laddr, UINT entry, int ucrw);
 
+/* paging */
+static UINT32 MEMCALL paging(UINT32 laddr, int ucrw);
 
-DWORD MEMCALL
-cpu_linear_memory_read(DWORD laddr, DWORD length, int code)
+/*
+ * linear memory access
+ */
+/* RMW */
+UINT8 MEMCALL
+cpu_memory_access_la_RMW_b(UINT32 laddr, UINT32 (CPUCALL *func)(UINT32, void *), void *arg)
 {
-	DWORD paddr;
-	DWORD remain;	/* page remain */
-	DWORD r;
-	DWORD value = 0;
-	int crw;
-	int pl;
-
-	crw = CPU_PAGING_PAGE_READ;
-	crw |= code ? CPU_PAGING_PAGE_CODE : CPU_PAGING_PAGE_DATA;
-	pl = (CPU_STAT_CPL == 3);
+	const int ucrw = CPU_PAGE_WRITE_DATA | CPU_STAT_USER_MODE;
+	struct tlb_entry *ep;
+	UINT32 paddr;
+	UINT32 result;
+	UINT8 value;
+
+	/* TLB */
+	ep = tlb_lookup(laddr, ucrw);
+	if (ep != NULL) {
+		paddr = ep->paddr + (laddr & CPU_PAGE_MASK);
+		goto onepage;
+	}
 
-	/* XXX: 4MB pages... */
-	remain = 0x1000 - (laddr & 0x00000fff);
-	for (;;) {
-		paddr = paging(laddr, crw, pl);
+	/* paging */
+	paddr = paging(laddr, ucrw);
+onepage:
+	value = cpu_memoryread(paddr);
+	result = (*func)(value, arg);
+	cpu_memorywrite(paddr, (UINT8)result);
+	return value;
+}
 
-		r = (remain > length) ? length : remain;
-		switch (r) {
-		case 1:
-			value = (value << 8) | cpu_memoryread(paddr);
-			break;
+UINT16 MEMCALL
+cpu_memory_access_la_RMW_w(UINT32 laddr, UINT32 (CPUCALL *func)(UINT32, void *), void *arg)
+{
+	const int ucrw = CPU_PAGE_WRITE_DATA | CPU_STAT_USER_MODE;
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT32 result;
+	UINT16 value;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		if ((laddr + 1) & CPU_PAGE_MASK)
+			goto onepage;
+
+		ep[1] = tlb_lookup(laddr + 1, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr + (laddr & CPU_PAGE_MASK);
+			goto separate;
+		}
+	}
 
-		case 2:
-			value = (value << 16) | cpu_memoryread_w(paddr);
-			break;
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	if ((laddr + 1) & CPU_PAGE_MASK) {
+onepage:
+		value = cpu_memoryread_w(paddr[0]);
+		result = (*func)(value, arg);
+		cpu_memorywrite_w(paddr[0], (UINT16)result);
+		return value;
+	}
 
-		case 3:
-			value <<= 24;
-			value |= cpu_memoryread_w(paddr) << 8;
-			value |= cpu_memoryread(paddr + 2);
-			break;
+	paddr[1] = paging(laddr + 1, ucrw);
+separate:
+	value = cpu_memoryread_b(paddr[0]);
+	value += (UINT16)cpu_memoryread_b(paddr[1]) << 8;
+	result = (*func)(value, arg);
+	cpu_memorywrite(paddr[0], (UINT8)result);
+	cpu_memorywrite(paddr[1], (UINT8)(result >> 8));
+	return value;
+}
 
-		case 4:
-			value = cpu_memoryread_d(paddr);
-			break;
+UINT32 MEMCALL
+cpu_memory_access_la_RMW_d(UINT32 laddr, UINT32 (CPUCALL *func)(UINT32, void *), void *arg)
+{
+	const int ucrw = CPU_PAGE_WRITE_DATA | CPU_STAT_USER_MODE;
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT32 result;
+	UINT32 value;
+	int remain;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= 4)
+			goto onepage;
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr + (laddr & CPU_PAGE_MASK);
+			goto separate;
+		}
+	}
 
-		default:
-			ia32_panic("cpu_linear_memory_read(): out of range\n");
-			break;
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= 4) {
+onepage:
+		value = cpu_memoryread_d(paddr[0]);
+		result = (*func)(value, arg);
+		cpu_memorywrite_d(paddr[0], result);
+		return value;
+	}
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	switch (remain) {
+	case 3:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[0] + 1) << 8;
+		value += (UINT32)cpu_memoryread(paddr[1]) << 24;
+		result = (*func)(value, arg);
+		cpu_memorywrite(paddr[0], (UINT8)result);
+		cpu_memorywrite_w(paddr[0] + 1, (UINT16)(result >> 8));
+		cpu_memorywrite(paddr[1], (UINT8)(result >> 24));
+		break;
+
+	case 2:
+		value = cpu_memoryread_w(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[1]) << 16;
+		result = (*func)(value, arg);
+		cpu_memorywrite_w(paddr[0], (UINT16)result);
+		cpu_memorywrite_w(paddr[1], (UINT16)(result >> 16));
+		break;
+
+	case 1:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[1]) << 8;
+		value += (UINT32)cpu_memoryread(paddr[1] + 2) << 24;
+		result = (*func)(value, arg);
+		cpu_memorywrite(paddr[0], (UINT8)result);
+		cpu_memorywrite_w(paddr[1], (UINT16)(result >> 8));
+		cpu_memorywrite(paddr[1] + 2, (UINT8)(result >> 24));
+		break;
+
+	default:
+		ia32_panic("cpu_memory_access_la_RMW_d: out of range (remain=%d)\n", remain);
+		value = 0;	/* XXX compiler happy */
+		break;
+	}
+	return value;
+}
+
+/* read */
+UINT8 MEMCALL
+cpu_linear_memory_read_b(UINT32 laddr, int ucrw)
+{
+	struct tlb_entry *ep;
+	UINT32 paddr;
+
+	/* TLB */
+	ep = tlb_lookup(laddr, ucrw);
+	if (ep != NULL) {
+		paddr = ep->paddr + (laddr & CPU_PAGE_MASK);
+		return cpu_memoryread(paddr);
+	}
+
+	/* paging */
+	paddr = paging(laddr, ucrw);
+	return cpu_memoryread(paddr);
+}
+
+UINT16 MEMCALL
+cpu_linear_memory_read_w(UINT32 laddr, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT16 value;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		if ((laddr + 1) & CPU_PAGE_MASK)
+			return cpu_memoryread_w(paddr[0]);
+
+		ep[1] = tlb_lookup(laddr + 1, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
 		}
+	}
 
-		if (length == r)
-			break;
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	if ((laddr + 1) & CPU_PAGE_MASK)
+		return cpu_memoryread_w(paddr[0]);
+
+	paddr[1] = paging(laddr + 1, ucrw);
+separate:
+	value = cpu_memoryread_b(paddr[0]);
+	value += (UINT16)cpu_memoryread_b(paddr[1]) << 8;
+	return value;
+}
 
-		length -= r;
-		remain -= r;
-		laddr += r;
+UINT32 MEMCALL
+cpu_linear_memory_read_d(UINT32 laddr, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT32 value;
+	UINT remain;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value))
+			return cpu_memoryread_d(paddr[0]);
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
+
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value))
+		return cpu_memoryread_d(paddr[0]);
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	switch (remain) {
+	case 3:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[0] + 1) << 8;
+		value += (UINT32)cpu_memoryread(paddr[1]) << 24;
+		break;
+
+	case 2:
+		value = cpu_memoryread_w(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[1]) << 16;
+		break;
+
+	case 1:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT32)cpu_memoryread_w(paddr[1]) << 8;
+		value += (UINT32)cpu_memoryread(paddr[1] + 2) << 24;
+		break;
+
+	default:
+		ia32_panic("cpu_linear_memory_read_d: out of range (remain=%d)\n", remain);
+		value = 0;	/* XXX compiler happy */
+		break;
+	}
+	return value;
+}
+
+UINT64 MEMCALL
+cpu_linear_memory_read_q(UINT32 laddr, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT64 value;
+	UINT remain;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value))
+			return cpu_memoryread_d(paddr[0]);
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
 	}
 
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value))
+		return cpu_memoryread_q(paddr[0]);
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	switch (remain) {
+	case 7:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT64)cpu_memoryread_w(paddr[0] + 1) << 8;
+		value += (UINT64)cpu_memoryread_d(paddr[0] + 3) << 24;
+		value += (UINT64)cpu_memoryread(paddr[1]) << 56;
+		break;
+
+	case 6:
+		value = cpu_memoryread_w(paddr[0]);
+		value += (UINT64)cpu_memoryread_d(paddr[0] + 2) << 16;
+		value += (UINT64)cpu_memoryread_w(paddr[1]) << 48;
+		break;
+
+	case 5:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT64)cpu_memoryread_d(paddr[0] + 1) << 8;
+		value += (UINT64)cpu_memoryread_w(paddr[1]) << 40;
+		value += (UINT64)cpu_memoryread(paddr[1] + 2) << 56;
+		break;
+
+	case 4:
+		value = cpu_memoryread_d(paddr[0]);
+		value += (UINT64)cpu_memoryread_d(paddr[1]) << 32;
+		break;
+
+	case 3:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT64)cpu_memoryread_w(paddr[0] + 1) << 8;
+		value += (UINT64)cpu_memoryread_d(paddr[1]) << 24;
+		value += (UINT64)cpu_memoryread(paddr[1] + 4) << 56;
+		break;
+
+	case 2:
+		value = cpu_memoryread_w(paddr[0]);
+		value += (UINT64)cpu_memoryread_d(paddr[1]) << 16;
+		value += (UINT64)cpu_memoryread_w(paddr[1] + 4) << 48;
+		break;
+
+	case 1:
+		value = cpu_memoryread(paddr[0]);
+		value += (UINT64)cpu_memoryread_d(paddr[1]) << 8;
+		value += (UINT64)cpu_memoryread_w(paddr[1] + 4) << 40;
+		value += (UINT64)cpu_memoryread(paddr[1] + 6) << 56;
+		break;
+
+	default:
+		ia32_panic("cpu_linear_memory_read_q: out of range (remain=%d)\n", remain);
+		value = 0;	/* XXX compiler happy */
+		break;
+	}
 	return value;
 }
 
+REG80 MEMCALL
+cpu_linear_memory_read_f(UINT32 laddr, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	REG80 value;
+	UINT remain;
+	UINT i, j;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value))
+			return cpu_memoryread_f(paddr[0]);
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
+
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value))
+		return cpu_memoryread_f(paddr[0]);
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	for (i = 0; i < remain; ++i) {
+		value.b[i] = cpu_memoryread(paddr[0] + i);
+	}
+	for (j = 0; i < 10; ++i, ++j) {
+		value.b[i] = cpu_memoryread(paddr[1] + j);
+	}
+	return value;
+}
+
+/* write */
 void MEMCALL
-cpu_linear_memory_write(DWORD laddr, DWORD length, DWORD value)
+cpu_linear_memory_write_b(UINT32 laddr, UINT8 value, int ucrw)
 {
-	DWORD paddr;
-	DWORD remain;	/* page remain */
-	DWORD r;
-	int pl;
+	struct tlb_entry *ep;
+	UINT32 paddr;
 
-	pl = (CPU_STAT_CPL == 3);
+	/* TLB */
+	ep = tlb_lookup(laddr, ucrw);
+	if (ep != NULL) {
+		paddr = ep->paddr + (laddr & CPU_PAGE_MASK);
+		cpu_memorywrite(paddr, value);
+		return;
+	}
 
-	/* XXX: 4MB pages... */
-	remain = 0x1000 - (laddr & 0x00000fff);
-	for (;;) {
-		paddr = paging(laddr, CPU_PAGING_PAGE_WRITE|CPU_PAGING_PAGE_DATA, pl);
+	/* paging */
+	paddr = paging(laddr, ucrw);
+	cpu_memorywrite(paddr, value);
+}
 
-		r = (remain > length) ? length : remain;
-		switch (r) {
-		case 1:
-			cpu_memorywrite(paddr, value);
-			value >>= 8;
-			break;
+void MEMCALL
+cpu_linear_memory_write_w(UINT32 laddr, UINT16 value, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
 
-		case 2:
-			cpu_memorywrite_w(paddr, value);
-			value >>= 16;
-			break;
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		if ((laddr + 1) & CPU_PAGE_MASK) {
+			cpu_memorywrite_w(paddr[0], value);
+			return;
+		}
 
-		case 3:
-			cpu_memorywrite_w(paddr, value);
-			cpu_memorywrite(paddr, value >> 16);
-			value >>= 24;
-			break;
+		ep[1] = tlb_lookup(laddr + 1, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
 
-		case 4:
-			cpu_memorywrite_d(paddr, value);
-			break;
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	if ((laddr + 1) & CPU_PAGE_MASK) {
+		cpu_memorywrite_w(paddr[0], value);
+		return;
+	}
 
-		default:
-			ia32_panic("cpu_linear_memory_write(): out of range\n");
-			break;
+	paddr[1] = paging(laddr + 1, ucrw);
+separate:
+	cpu_memorywrite(paddr[0], (UINT8)value);
+	cpu_memorywrite(paddr[1], (UINT8)(value >> 8));
+}
+
+void MEMCALL
+cpu_linear_memory_write_d(UINT32 laddr, UINT32 value, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT remain;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value)) {
+			cpu_memorywrite_d(paddr[0], value);
+			return;
 		}
 
-		if (length == r)
-			break;
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
+
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value)) {
+		cpu_memorywrite_d(paddr[0], value);
+		return;
+	}
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	switch (remain) {
+	case 3:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_w(paddr[0] + 1, (UINT16)(value >> 8));
+		cpu_memorywrite(paddr[1], (UINT8)(value >> 24));
+		break;
+
+	case 2:
+		cpu_memorywrite_w(paddr[0], (UINT16)value);
+		cpu_memorywrite_w(paddr[1], (UINT16)(value >> 16));
+		break;
+
+	case 1:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_w(paddr[1], (UINT16)(value >> 8));
+		cpu_memorywrite(paddr[1] + 2, (UINT8)(value >> 24));
+		break;
+
+	default:
+		ia32_panic("cpu_linear_memory_write_d: out of range (remain=%d)\n", remain);
+		break;
+	}
+}
+
+void MEMCALL
+cpu_linear_memory_write_q(UINT32 laddr, UINT64 value, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT remain;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value)) {
+			cpu_memorywrite_q(paddr[0], value);
+			return;
+		}
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
+
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value)) {
+		cpu_memorywrite_q(paddr[0], value);
+		return;
+	}
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	switch (remain) {
+	case 7:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_w(paddr[0] + 1, (UINT16)(value >> 8));
+		cpu_memorywrite_d(paddr[0] + 3, (UINT32)(value >> 24));
+		cpu_memorywrite(paddr[1], (UINT8)(value >> 56));
+		break;
+
+	case 6:
+		cpu_memorywrite_w(paddr[0], (UINT16)value);
+		cpu_memorywrite_d(paddr[0] + 2, (UINT32)(value >> 16));
+		cpu_memorywrite_w(paddr[1], (UINT16)(value >> 48));
+		break;
+
+	case 5:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_d(paddr[0] + 1, (UINT32)(value >> 8));
+		cpu_memorywrite_w(paddr[1], (UINT16)(value >> 40));
+		cpu_memorywrite(paddr[1] + 2, (UINT8)(value >> 56));
+		break;
+
+	case 4:
+		cpu_memorywrite_d(paddr[0], (UINT32)value);
+		cpu_memorywrite_d(paddr[1], (UINT32)(value >> 32));
+		break;
+
+	case 3:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_w(paddr[0] + 1, (UINT16)(value >> 8));
+		cpu_memorywrite_d(paddr[1], (UINT32)(value >> 24));
+		cpu_memorywrite(paddr[1] + 4, (UINT8)(value >> 56));
+		break;
+
+	case 2:
+		cpu_memorywrite_w(paddr[0], (UINT16)value);
+		cpu_memorywrite_d(paddr[1], (UINT32)(value >> 16));
+		cpu_memorywrite_w(paddr[1] + 4, (UINT16)(value >> 48));
+		break;
+
+	case 1:
+		cpu_memorywrite(paddr[0], (UINT8)value);
+		cpu_memorywrite_d(paddr[1], (UINT32)(value >> 8));
+		cpu_memorywrite_w(paddr[1] + 4, (UINT16)(value >> 40));
+		cpu_memorywrite(paddr[1] + 6, (UINT8)(value >> 56));
+		break;
+
+	default:
+		ia32_panic("cpu_linear_memory_write_q: out of range (remain=%d)\n", remain);
+		break;
+	}
+}
+
+void MEMCALL
+cpu_linear_memory_write_f(UINT32 laddr, const REG80 *value, int ucrw)
+{
+	struct tlb_entry *ep[2];
+	UINT32 paddr[2];
+	UINT remain;
+	UINT i, j;
+
+	/* TLB */
+	ep[0] = tlb_lookup(laddr, ucrw);
+	if (ep[0] != NULL) {
+		paddr[0] = ep[0]->paddr + (laddr & CPU_PAGE_MASK);
+		remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+		if (remain >= sizeof(value)) {
+			cpu_memorywrite_f(paddr[0], value);
+			return;
+		}
+
+		ep[1] = tlb_lookup(laddr + remain, ucrw);
+		if (ep[1] != NULL) {
+			paddr[1] = ep[1]->paddr;
+			goto separate;
+		}
+	}
+
+	/* paging */
+	paddr[0] = paging(laddr, ucrw);
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	if (remain >= sizeof(value)) {
+		cpu_memorywrite_f(paddr[0], value);
+		return;
+	}
+
+	paddr[1] = paging(laddr + remain, ucrw);
+separate:
+	for (i = 0; i < remain; ++i) {
+		cpu_memorywrite(paddr[0] + i, value->b[i]);
+	}
+	for (j = 0; i < 10; ++i, ++j) {
+		cpu_memorywrite(paddr[1] + j, value->b[i]);
+	}
+}
+
+/*
+ * linear address memory access function
+ */
+void MEMCALL
+cpu_memory_access_la_region(UINT32 laddr, UINT length, int ucrw, UINT8 *data)
+{
+	UINT32 paddr;
+	UINT remain;	/* page remain */
+	UINT r;
+
+	if (length == 0)
+		return;
+
+	remain = CPU_PAGE_SIZE - (laddr & CPU_PAGE_MASK);
+	for (;;) {
+		if (!CPU_STAT_PAGING) {
+			paddr = laddr;
+		} else {
+			paddr = paging(laddr, ucrw);
+		}
+
+		r = (remain > length) ? length : remain;
+		if (!(ucrw & CPU_PAGE_WRITE)) {
+			cpu_memoryread_region(paddr, data, r);
+		} else {
+			cpu_memorywrite_region(paddr, data, r);
+		}
 
 		length -= r;
-		remain -= r;
+		if (length == 0)
+			break;
+
+		data += r;
 		laddr += r;
+		remain -= r;
+		if (remain <= 0) {
+			/* next page */
+			remain += CPU_PAGE_SIZE;
+		}
 	}
 }
 
-void MEMCALL
-paging_check(DWORD laddr, DWORD length, int rw)
+UINT32 MEMCALL
+laddr2paddr(UINT32 laddr, int ucrw)
 {
-	DWORD addr;
-	int n;
-	int pl;
 
-	pl = (CPU_STAT_CPL == 3);
+	return paging(laddr, ucrw);
+}
 
-	/* XXX: 4MB pages... */
-	n = ((laddr & 0xfff) + length) / 0x1000;
-	addr = (laddr & ~0xfff);
-	do {
-		(void)paging(addr, rw, pl);
-		addr += 0x1000;
-	} while (--n > 0);
-}
-
-static DWORD
-paging(DWORD laddr, int crw, int user_mode)
-{
-	DWORD paddr;	/* physical address */
-	DWORD pde_addr;	/* page directory entry address */
-	DWORD pde;	/* page directory entry */
-	DWORD pte_addr;	/* page table entry address */
-	DWORD pte;	/* page table entry */
-	DWORD bit;
-	DWORD err = 0;
-
-#if defined(IA32_SUPPORT_TLB)
-	if (tlb_lookup(laddr, crw, &paddr))
-		return paddr;
-#endif	/* IA32_SUPPORT_TLB */
+/*
+ * paging
+ */
+static UINT32 MEMCALL
+paging(UINT32 laddr, int ucrw)
+{
+	UINT32 paddr;		/* physical address */
+	UINT32 pde_addr;	/* page directory entry address */
+	UINT32 pde;		/* page directory entry */
+	UINT32 pte_addr;	/* page table entry address */
+	UINT32 pte;		/* page table entry */
+	UINT bit;
+	UINT err;
+	struct tlb_entry *ep;
+
+	ep = tlb_lookup(laddr, ucrw);
+	if (ep != NULL)
+		return ep->paddr + (laddr & CPU_PAGE_MASK);
 
-	pde_addr = (CPU_CR3 & CPU_CR3_PD_MASK) | ((laddr >> 20) & 0xffc);
+	pde_addr = CPU_STAT_PDE_BASE + ((laddr >> 20) & 0xffc);
 	pde = cpu_memoryread_d(pde_addr);
 	if (!(pde & CPU_PDE_PRESENT)) {
-		VERBOSE(("PDE is not present. (laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x)", laddr, pde_addr, pde));
+		VERBOSE(("paging: PTE page is not present"));
+		VERBOSE(("paging: CPU_CR3 = 0x%08x", CPU_CR3));
+		VERBOSE(("paging: laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x", laddr, pde_addr, pde));
 		err = 0;
 		goto pf_exception;
 	}
@@ -430,40 +902,26 @@ paging(DWORD laddr, int crw, int user_mo
 		cpu_memorywrite_d(pde_addr, pde);
 	}
 
-	/* no support PAE */
-	__ASSERT(!(CPU_CR4 & CPU_CR4_PAE));
-
-	if ((CPU_CR4 & CPU_CR4_PSE) && (pde & CPU_PDE_PAGE_SIZE)) {
-		/* 4MB page size */
-
-		/* fake PTE bit */
-		pte = pde | CPU_PTE_DIRTY;
-		pte_addr = 0;	/* compiler happy */
-
-		/* make physical address */
-		paddr = (pde & CPU_PDE_4M_BASEADDR_MASK) | (laddr & 0x003fffff);
-	} else {
-		/* 4KB page size */
-		pte_addr = (pde & CPU_PDE_BASEADDR_MASK) | ((laddr >> 10) & 0xffc);
-		pte = cpu_memoryread_d(pte_addr);
-		if (!(pte & CPU_PTE_PRESENT)) {
-			VERBOSE(("PTE is not present. (laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x, pte_addr = 0x%08x, pte = 0x%08x)", laddr, pde_addr, pde, pte_addr, pte));
-			err = 0;
-			goto pf_exception;
-		}
-		if (!(pte & CPU_PTE_ACCESS)) {
-			pte |= CPU_PTE_ACCESS;
-			cpu_memorywrite_d(pte_addr, pte);
-		}
-
-		/* make physical address */
-		paddr = (pte & CPU_PTE_BASEADDR_MASK) | (laddr & 0x00000fff);
+	pte_addr = (pde & CPU_PDE_BASEADDR_MASK) + ((laddr >> 10) & 0xffc);
+	pte = cpu_memoryread_d(pte_addr);
+	if (!(pte & CPU_PTE_PRESENT)) {
+		VERBOSE(("paging: page is not present"));
+		VERBOSE(("paging: laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x", laddr, pde_addr, pde));
+		VERBOSE(("paging: pte_addr = 0x%08x, pte = 0x%08x", pte_addr, pte));
+		err = 0;
+		goto pf_exception;
+	}
+	if (!(pte & CPU_PTE_ACCESS)) {
+		pte |= CPU_PTE_ACCESS;
+		cpu_memorywrite_d(pte_addr, pte);
 	}
 
-	bit  = crw & 1;
+	/* make physical address */
+	paddr = (pte & CPU_PTE_BASEADDR_MASK) + (laddr & CPU_PAGE_MASK);
+
+	bit  = ucrw & (CPU_PAGE_WRITE|CPU_PAGE_USER_MODE);
 	bit |= (pde & pte & (CPU_PTE_WRITABLE|CPU_PTE_USER_MODE));
-	bit |= (user_mode << 3);
-	bit |= (CPU_CR0 & CPU_CR0_WP) >> 12;
+	bit |= CPU_STAT_WP;
 
 #if !defined(USE_PAGE_ACCESS_TABLE)
 	if (!(page_access & (1 << bit)))
@@ -471,297 +929,165 @@ paging(DWORD laddr, int crw, int user_mo
 	if (!(page_access_bit[bit]))
 #endif
 	{
-		VERBOSE(("page access violation. (laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x, pte_addr = 0x%08x, pte = 0x%08x, paddr = 0x%08x, bit = 0x%08x)", laddr, pde_addr, pde, pte_addr, pte, paddr, bit));
+		VERBOSE(("paging: page access violation."));
+		VERBOSE(("paging: laddr = 0x%08x, pde_addr = 0x%08x, pde = 0x%08x", laddr, pde_addr, pde));
+		VERBOSE(("paging: pte_addr = 0x%08x, pte = 0x%08x", pte_addr, pte));
+		VERBOSE(("paging: paddr = 0x%08x, bit = 0x%08x", paddr, bit));
 		err = 1;
 		goto pf_exception;
 	}
 
-	if ((crw & 1) && !(pte & CPU_PTE_DIRTY)) {
+	if ((ucrw & CPU_PAGE_WRITE) && !(pte & CPU_PTE_DIRTY)) {
 		pte |= CPU_PTE_DIRTY;
 		cpu_memorywrite_d(pte_addr, pte);
 	}
 
-#if defined(IA32_SUPPORT_TLB)
-	tlb_update(paddr, pte, crw);
-#endif	/* IA32_SUPPORT_TLB */
+	tlb_update(laddr, pte, (bit & (CPU_PTE_WRITABLE|CPU_PTE_USER_MODE)) + ((ucrw & CPU_PAGE_CODE) ? 1 : 0));
 
 	return paddr;
 
 pf_exception:
 	CPU_CR2 = laddr;
-	err |= ((crw & 1) << 1) | (user_mode << 2);
+	err |= (ucrw & CPU_PAGE_WRITE) << 1;
+	err |= (ucrw & CPU_PAGE_USER_MODE) >> 1;
 	EXCEPTION(PF_EXCEPTION, err);
 	return 0;	/* compiler happy */
 }
 
-#if defined(IA32_SUPPORT_TLB)
 /* 
  * TLB
  */
-typedef struct {
-	BYTE	valid;	/* TLB entry is valid */
-	BYTE	global;	/* this TLB entry is global */
-	BYTE	score;
-	BYTE	pad;
-
-	DWORD	tag;
-	DWORD	mask;	/* 4K or 2M or 4M */
+#define	TLB_TAG_SHIFT		TLB_ENTRY_TAG_MAX_SHIFT
+#define	TLB_TAG_MASK		(~((1 << TLB_TAG_SHIFT) - 1))
+#define	TLB_GET_TAG_ADDR(ep)	((ep)->tag & TLB_TAG_MASK)
+#define	TLB_SET_TAG_ADDR(ep, addr) \
+do { \
+	(ep)->tag &= ~TLB_TAG_MASK; \
+	(ep)->tag |= (addr) & TLB_TAG_MASK; \
+} while (/*CONSTCOND(*/ 0)
+
+#define	TLB_IS_VALID(ep)	((ep)->tag & TLB_ENTRY_TAG_VALID)
+#define	TLB_SET_VALID(ep)	((ep)->tag = TLB_ENTRY_TAG_VALID)
+#define	TLB_SET_INVALID(ep)	((ep)->tag = 0)
+
+#define	TLB_IS_WRITABLE(ep)	((ep)->tag & CPU_PTE_WRITABLE)
+#define	TLB_IS_USERMODE(ep)	((ep)->tag & CPU_PTE_USER_MODE)
+#define	TLB_IS_DIRTY(ep)	((ep)->tag & TLB_ENTRY_TAG_DIRTY)
+#if (CPU_FEATURES & CPU_FEATURE_PGE) == CPU_FEATURE_PGE
+#define	TLB_IS_GLOBAL(ep)	((ep)->tag & TLB_ENTRY_TAG_GLOBAL)
+#else
+#define	TLB_IS_GLOBAL(ep)	0
+#endif
 
-	DWORD	paddr;	/* physical addr */
-} TLB_ENTRY_T;
+#define	TLB_SET_TAG_FLAGS(ep, entry, bit) \
+do { \
+	(ep)->tag |= (entry) & (CPU_PTE_GLOBAL_PAGE|CPU_PTE_DIRTY); \
+	(ep)->tag |= (bit) & (CPU_PTE_WRITABLE|CPU_PTE_USER_MODE); \
+} while (/*CONSTCOND*/ 0)
+
+#define	NTLB		2	/* 0: DTLB, 1: ITLB */
+#define	NENTRY		(1 << 6)
+#define	TLB_ENTRY_SHIFT	12
+#define	TLB_ENTRY_MASK	(NENTRY - 1)
 
 typedef struct {
-	BYTE		kind;
-#define	TLB_KIND_INSTRUCTION	(1 << 1)
-#define	TLB_KIND_DATA		(1 << 2)
-#define	TLB_KIND_COMBINE	(TLB_KIND_INSTRUCTION|TLB_KIND_DATA)
-#define	TLB_KIND_SMALL		(1 << 3)
-#define	TLB_KIND_LARGE		(1 << 4)
-#define	TLB_KIND_BOTH		(TLB_KIND_SMALL|TLB_KIND_LARGE)
-
-	BYTE		way;	/* n-way associative */
-	BYTE		idx;	/* number of TLB index */
-	BYTE		bpad;
-
-	WORD		num;	/* number of TLB entry */
-	WORD		wpad;
-
-	TLB_ENTRY_T*	entry;	/* entry[assoc][idx] or entry[assoc] if idx == 1*/
-} TLB_T;
-
-static int ntlb;
-static TLB_T tlb[4];	/* i TLB, i (lp) TLB, d TLB, d (lp) TLB */
-
-#if defined(IA32_PROFILE_TLB)
-/* profiling */
-static DWORD tlb_hits;
-static DWORD tlb_misses;
-static DWORD tlb_lookups;
-static DWORD tlb_updates;
-static DWORD tlb_flushes;
-static DWORD tlb_global_flushes;
-static DWORD tlb_entry_flushes;
-
-#define	PROFILE_INC(v)	(v)++;
-#else	/* !IA32_PROFILE_TLB */
-#define	PROFILE_INC(v)
-#endif	/* IA32_PROFILE_TLB */
+	struct tlb_entry entry[NENTRY];
+} tlb_t;
+static tlb_t tlb[NTLB];
 
 void
-tlb_init()
+tlb_init(void)
 {
-	int i;
 
-	for (i = 0; i < NELEMENTS(tlb); i++) {
-		if (tlb[i].entry) {
-			free(tlb[i].entry);
-		}
-	}
 	memset(tlb, 0, sizeof(tlb));
-
-#if defined(IA32_PROFILE_TLB)
-	tlb_hits = 0;
-	tlb_misses = 0;
-	tlb_lookups = 0;
-	tlb_updates = 0;
-	tlb_flushes = 0;
-	tlb_global_flushes = 0;
-	tlb_entry_flushes = 0;
-#endif	/* IA32_PROFILE_TLB */
-
-	/* XXX ’仿’𤪦扔’氾’左扑𦾟白﹜甲﹜平﹜末﹜乓﹜氾﹜瓦 TLB 弗旦正亦﹜罋衛菑馱*/
-
-	/* 午曰丐尹內 i486 溥摯匹＿ */
-	/* combine (I/D) TLB: 4KB Pages, 4-way set associative 32 entries */
-	ntlb = 1;
-	tlb[0].kind = TLB_KIND_COMBINE | TLB_KIND_SMALL;
-	tlb[0].num = 32;
-	tlb[0].way = 4;
-	tlb[0].idx = tlb[0].num / tlb[0].way;
-
-	for (i = 0; i < ntlb; i++) {
-		tlb[i].entry = (TLB_ENTRY_T*)calloc(sizeof(TLB_ENTRY_T), tlb[i].num);
-		if (tlb[i].entry == 0) {
-			ia32_panic("tlb_init(): can't alloc TLB entry\n");
-		}
-	}
 }
 
-void
+void MEMCALL
 tlb_flush(BOOL allflush)
 {
-	TLB_ENTRY_T* ep;
-	int i, j;
+	struct tlb_entry *ep;
+	int i;
+	int n;
 
 	if (allflush) {
-		PROFILE_INC(tlb_global_flushes);
-	} else {
-		PROFILE_INC(tlb_flushes);
+		tlb_init();
+		return;
 	}
 
-	for (i = 0; i < ntlb; i++) {
-		ep = tlb[i].entry;
-		for (j = 0; j < tlb[i].num; j++, ep++) {
-			if (ep->valid && (allflush || !ep->global)) {
-				ep->valid = 0;
-				PROFILE_INC(tlb_entry_flushes);
+	for (n = 0; n < NTLB; n++) {
+		for (i = 0; i < NENTRY ; i++) {
+			ep = &tlb[n].entry[i];
+			if (TLB_IS_VALID(ep) && !TLB_IS_GLOBAL(ep)) {
+				TLB_SET_INVALID(ep);
 			}
 		}
 	}
 }
 
-void
-tlb_flush_page(DWORD vaddr)
+void MEMCALL
+tlb_flush_page(UINT32 laddr)
 {
-	TLB_ENTRY_T* ep;
+	struct tlb_entry *ep;
 	int idx;
-	int i;
+	int n;
 
-	for (i = 0; i < ntlb; i++) {
-		if (tlb[i].idx == 1) {
-			/* fully set associative */
-			idx = 0;
-		} else {
-			if (tlb[i].kind & TLB_KIND_SMALL) {
-				idx = (vaddr >> 12) & (tlb[i].idx - 1);
-			} else {
-				idx = (vaddr >> 22) & (tlb[i].idx - 1);
-			}
-		}
+	idx = (laddr >> TLB_ENTRY_SHIFT) & TLB_ENTRY_MASK;
 
-		/* search */
-		ep = &tlb[i].entry[idx * tlb[i].way];
-		for (i = 0; i < tlb[i].way; i++) {
-			if (ep->valid) {
-				if ((vaddr & ep->mask) == ep->tag) {
-					ep->valid = 0;
-					PROFILE_INC(tlb_entry_flushes);
-					break;
-				}
+	for (n = 0; n < NTLB; n++) {
+		ep = &tlb[n].entry[idx];
+		if (TLB_IS_VALID(ep)) {
+			if ((laddr & TLB_TAG_MASK) == TLB_GET_TAG_ADDR(ep)) {
+				TLB_SET_INVALID(ep);
 			}
 		}
 	}
 }
 
-static BOOL
-tlb_lookup(DWORD laddr, int crw, DWORD* paddr)
+struct tlb_entry * MEMCALL
+tlb_lookup(UINT32 laddr, int ucrw)
 {
-	TLB_ENTRY_T* ep;
+	struct tlb_entry *ep;
+	UINT bit;
 	int idx;
-	int i;
-
-	PROFILE_INC(tlb_lookups);
-
-	crw &= CPU_PAGING_PAGE_CODE | CPU_PAGING_PAGE_DATA;
-	for (i = 0; i < ntlb; i++) {
-		if (tlb[i].kind & crw) {
-			if (tlb[i].idx == 1) {
-				/* fully set associative */
-				idx = 0;
-			} else {
-				if (tlb[i].kind & TLB_KIND_SMALL) {
-					idx = (laddr >> 12) & (tlb[i].idx - 1);
-				} else {
-					idx = (laddr >> 22) & (tlb[i].idx - 1);
-				}
-			}
+	int n;
 
-			/* search */
-			ep = &tlb[i].entry[idx * tlb[i].way];
-			for (i = 0; i < tlb[i].way; i++) {
-				if (ep->valid) {
-					if ((laddr & ep->mask) == ep->tag) {
-						if (ep->score != (BYTE)~0)
-							ep->score++;
-						*paddr = ep->paddr;
-						PROFILE_INC(tlb_hits);
-						return TRUE;
-					}
+	n = (ucrw & CPU_PAGE_CODE) ? 1 : 0;
+	idx = (laddr >> TLB_ENTRY_SHIFT) & TLB_ENTRY_MASK;
+	ep = &tlb[n].entry[idx];
+
+	if (TLB_IS_VALID(ep)) {
+		if ((laddr & TLB_TAG_MASK) == TLB_GET_TAG_ADDR(ep)) {
+			bit = ucrw & (CPU_PAGE_WRITE|CPU_PAGE_USER_MODE);
+			bit |= ep->tag & (CPU_PTE_WRITABLE|CPU_PTE_USER_MODE);
+			bit |= CPU_STAT_WP;
+#if !defined(USE_PAGE_ACCESS_TABLE)
+			if ((page_access & (1 << bit)))
+#else
+			if (page_access_bit[bit])
+#endif
+			{
+				if (!(ucrw & CPU_PAGE_WRITE) || TLB_IS_DIRTY(ep)) {
+					return ep;
 				}
 			}
 		}
 	}
-	PROFILE_INC(tlb_misses);
-	return FALSE;
+	return NULL;
 }
 
-static void
-tlb_update(DWORD paddr, DWORD entry, int crw)
+static void MEMCALL
+tlb_update(UINT32 laddr, UINT entry, int bit)
 {
-	TLB_ENTRY_T* ep;
+	struct tlb_entry *ep;
 	int idx;
-	int i, j;
-	int min_way;
-	WORD min_score = ~0;
-
-	PROFILE_INC(tlb_updates);
-
-	crw &= CPU_PAGING_PAGE_CODE | CPU_PAGING_PAGE_DATA;
-	for (i = 0; i < ntlb; i++) {
-		if (tlb[i].kind & crw) {
-			if (tlb[i].idx == 1) {
-				/* fully set associative */
-				idx = 0;
-			} else {
-				/* n-way set associative */
-				if (!(entry & CPU_PDE_PAGE_SIZE)) {
-					if (!(tlb[i].kind & TLB_KIND_SMALL))
-						continue;
-					idx = (entry >> 12) & (tlb[i].idx - 1);
-				} else {
-					if (!(tlb[i].kind & TLB_KIND_LARGE))
-						continue;
-					idx = (entry >> 22) & (tlb[i].idx - 1);
-				}
-			}
-
-			/* search */
-			ep = &tlb[i].entry[idx * tlb[i].way];
-			for (min_way = 0, j = 0; j < tlb[i].way; j++, ep++) {
-				if (ep->valid) {
-					if (min_score >= ep->score) {
-						min_way = j;
-						min_score = ep->score;
-					}
-				} else {
-					min_way = j;
-					min_score = 0;
-					break;
-				}
-			}
-
-			/* replace */
-			ep = &tlb[i].entry[idx * tlb[i].way + min_way];
-			ep->valid = 1;
-			ep->global = (entry & CPU_PTE_GLOBAL_PAGE) ? 1 : 0;
-			ep->score = 0;
-			ep->mask = (entry & CPU_PDE_PAGE_SIZE) ?  CPU_PDE_4M_BASEADDR_MASK : CPU_PTE_BASEADDR_MASK;
-			ep->tag = entry & ep->mask;
-			ep->paddr = paddr;
-			break;
-		}
-	}
-	__ASSERT(i != ntlb);
-}
-#else	/* !IA32_SUPPORT_TLB */
-void
-tlb_init()
-{
-
-	/* nothing to do */
-}
-
-void
-tlb_flush(BOOL allflush)
-{
-
-	(void)allflush;
-}
-
-void
-tlb_flush_page(DWORD laddr)
-{
+	int n;
 
-	(void)laddr;
+	n = bit & 1;
+	idx = (laddr >> TLB_ENTRY_SHIFT) & TLB_ENTRY_MASK;
+	ep = &tlb[n].entry[idx];
+
+	TLB_SET_VALID(ep);
+	TLB_SET_TAG_ADDR(ep, laddr);
+	TLB_SET_TAG_FLAGS(ep, entry, bit);
+	ep->paddr = entry & CPU_PTE_BASEADDR_MASK;
 }
-#endif	/* IA32_SUPPORT_TLB */