[NTOS] hack together primitive windbg context switching

Co-authored-by: Justin Miller <justinmiller100@gmail.com>

boot/bootdata/livecd.ini

@@ -10,7 +10,7 @@ MinimalUI=Yes
 LiveCD="LiveCD"
 LiveCD_Debug="LiveCD (Debug)"
 LiveCD_Aacpi="LiveCD ACPI APIC (Debug)"
-LiveCD_VBoxDebug="LiveCD (VBox Debug)"
+LiveCD_Macpi="LiveCD SMP ACPI (Debug)"
 LiveCD_Screen="LiveCD (Screen)"
 LiveCD_LogFile="LiveCD (Log file)"
 
@@ -34,6 +34,19 @@ BootType=Windows2003
 SystemPath=\reactos
 Options=/DEBUG /DEBUGPORT=VBOX /SOS /MININT
 
+[LiveCD_Macpi]
+BootType=Windows2003
+SystemPath=\reactos
+Hal=HALMACPI.DLL
+Kernel=NTKRNLMP.EXE
+Options=/DEBUG /DEBUGPORT=COM1 /BAUDRATE=115200 /SOS /MININT
+
+[LiveCD_Aacpi]
+BootType=Windows2003
+SystemPath=\reactos
+Hal=HALAACPI.DLL
+Options=/DEBUG /DEBUGPORT=COM1 /BAUDRATE=115200 /SOS /MININT
+
 [LiveCD_Screen]
 BootType=Windows2003
 SystemPath=\reactos

boot/bootdata/txtsetup.sif

@@ -230,7 +230,7 @@ DefaultLanguage  = 00000409
 pci_up = "Standard PC Uniprocessor"
 ;pci_mp = "Standard PC Multiprocessor"
 acpi_up = "ACPI PC Uniprocessor"
-;acpi_mp = "ACPI PC Multiprocessor"
+acpi_mp = "ACPI PC Multiprocessor"
 apic_up = "Standard APIC PC Uniprocessor"
 aacpi_up = "ACPI APIC PC Uniprocessor"
 xbox = "Original Xbox (x86 based)"
@@ -245,7 +245,7 @@ x64_mp = "Standard x64 Multiprocessor"
 pci_up = "PC UP"
 ;pci_mp = "PC MP"
 acpi_up = "ACPI UP"
-;acpi_mp = "ACPI MP"
+acpi_mp = "ACPI MP"
 apic_up = "APIC UP"
 aacpi_up = "AAPIC UP"
 xbox = "Xbox"
@@ -267,9 +267,9 @@ hal.dll      = 1,,,,,,,2,,,,1,2
 ntoskrnl.exe = 1,,,,,,,2,,,,1,2
 halacpi.dll  = 1,,,,,,,2,,,hal.dll,1,2
 
-;[Files.acpi_mp]
-;ntkrnlmp.exe = 1,,,,,,,2,,,ntoskrnl.exe,1,2
-;halacpi.dll  = 1,,,,,,,2,,,hal.dll,1,2
+[Files.acpi_mp]
+ntkrnlmp.exe = 1,,,,,,,2,,,ntoskrnl.exe,1,2
+halmacpi.dll  = 1,,,,,,,2,,,hal.dll,1,2
 
 [Files.apic_up]
 ntoskrnl.exe = 1,,,,,,,2,,,,1,2

hal/halx86/CMakeLists.txt

@@ -46,7 +46,7 @@ include(acpi.cmake)
 include(apic.cmake)
 include(up.cmake)
 include(smp.cmake)
-
+include(apicsmp.cmake)
 if(ARCH STREQUAL "i386")
     include(pcidata.cmake)
     include(legacy.cmake)
@@ -66,8 +66,7 @@ if(ARCH STREQUAL "i386")
     add_hal(halxbox SOURCES xbox/halxbox.rc COMPONENTS xbox up)
     add_hal(halpc98 SOURCES pc98/halpc98.rc COMPONENTS pc98 up)
 
-    #add_hal(halmacpi SOURCES smp/halmacpi.rc COMPONENTS generic acpi smp apic)
-    #add_hal(halmp SOURCES mp/halmp.rc COMPONENTS generic legacy smp apic)
+    add_hal(halmacpi SOURCES smp/halmacpi.rc COMPONENTS generic acpi smp apicsmp)
 
 elseif(ARCH STREQUAL "amd64")
 

hal/halx86/apic/apic.c

@@ -532,6 +532,7 @@ HalpInitializePICs(IN BOOLEAN EnableInterrupts)
 
     /* Set interrupt handlers in the IDT */
     KeRegisterInterruptHandler(APIC_CLOCK_VECTOR, HalpClockInterrupt);
+    KeRegisterInterruptHandler(APIC_IPI_VECTOR, HalpIpiInterrupt);
 #ifndef _M_AMD64
     KeRegisterInterruptHandler(APC_VECTOR, HalpApcInterrupt);
     KeRegisterInterruptHandler(DISPATCH_VECTOR, HalpDispatchInterrupt);
@@ -540,6 +541,7 @@ HalpInitializePICs(IN BOOLEAN EnableInterrupts)
     /* Register the vectors for APC and dispatch interrupts */
     HalpRegisterVector(IDT_INTERNAL, 0, APC_VECTOR, APC_LEVEL);
     HalpRegisterVector(IDT_INTERNAL, 0, DISPATCH_VECTOR, DISPATCH_LEVEL);
+    HalpRegisterVector(IDT_INTERNAL, 0, APIC_IPI_VECTOR, IPI_LEVEL);
 
     /* Restore interrupt state */
     if (EnableInterrupts) EFlags |= EFLAGS_INTERRUPT_MASK;
@@ -639,6 +641,39 @@ HalpDispatchInterruptHandler(IN PKTRAP_FRAME TrapFrame)
     /* Exit the interrupt */
     KiEoiHelper(TrapFrame);
 }
+
+VOID
+FASTCALL
+HalpIpiInterruptHandler(IN PKTRAP_FRAME TrapFrame)
+{
+    KIRQL Irql;
+
+    /* Enter trap */
+    KiEnterInterruptTrap(TrapFrame);
+
+    /* Start the interrupt */
+    if (!HalBeginSystemInterrupt(IPI_LEVEL, APIC_IPI_VECTOR, &Irql))
+    {
+        /* Spurious, just end the interrupt */
+        KiEoiHelper(TrapFrame);
+    }
+
+    /* Raise to DISPATCH_LEVEL */
+    ApicRaiseIrql(IPI_LEVEL);
+
+    /* End the interrupt */
+    ApicSendEOI();
+
+    _enable();
+    KiIpiServiceRoutine(TrapFrame, NULL);
+    _disable();
+
+    /* Restore the old IRQL */
+    ApicLowerIrql(Irql);
+
+    /* Exit the interrupt */
+    KiEoiHelper(TrapFrame);
+}
 #endif
 
 

hal/halx86/apic/apicsmp.c

@@ -1,15 +1,16 @@
 /*
  * PROJECT:     ReactOS HAL
  * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
- * FILE:        hal/halx86/apic/apicsmp.c
  * PURPOSE:     SMP specific APIC code
- * PROGRAMMERS: Copyright 2021 Timo Kreuzer (timo.kreuzer@reactos.org)
+ * COPYRIGHT:   Copyright 2021 Timo Kreuzer (timo.kreuzer@reactos.org)
+ *              Copyright 2021 Justin Miller (justinmiller100@gmail.com)
  */
 
 /* INCLUDES *******************************************************************/
 
 #include <hal.h>
 #include "apicp.h"
+#include <smp.h>
 #define NDEBUG
 #include <debug.h>
 
@@ -84,13 +85,43 @@ ApicRequestGlobalInterrupt(
 
 /* SMP SUPPORT FUNCTIONS ******************************************************/
 
-// Should be called by SMP version of HalRequestIpi
 VOID
 NTAPI
 HalpRequestIpi(KAFFINITY TargetProcessors)
 {
-    UNIMPLEMENTED;
-    __debugbreak();
+    LONG i;
+    KAFFINITY Current;
+    /*
+    *
+    * CPU masking is done in software thanks to KAFFINITY
+    * - Destination is ALWAYS : APIC_DSH_Destination
+    * - Mode is ALWAYS dependdent on xAPIC+ or legacy APIC
+    * -
+    */
+   // DPRINT1("KAFFINITY is %X\n", TargetProcessors);
+
+    for (i = 0, Current = 1; i < KeNumberProcessors; i++, Current <<= 1)
+    {
+        if (TargetProcessors & Current)
+        {
+           // DPRINT1("Sending IPI to CPU: %X\n", i);
+            ApicRequestGlobalInterrupt(i, APIC_IPI_VECTOR, APIC_MT_Fixed,
+                APIC_TGM_Edge, APIC_DSH_Destination);
+        }
+    }
 }
 
-// APIC specific SMP code here
+VOID
+ApicStartApplicationProcessor(ULONG NTProcessorNumber, PHYSICAL_ADDRESS StartupLoc)
+{
+    /* Init IPI */
+    ApicRequestGlobalInterrupt(NTProcessorNumber, 0,
+        APIC_MT_INIT, APIC_TGM_Edge, APIC_DSH_Destination);
+
+    /* Stall execution for a bit to give APIC time */
+    KeStallExecutionProcessor(1000);
+
+    /* Startup IPI */
+    ApicRequestGlobalInterrupt(NTProcessorNumber, (StartupLoc.LowPart) >> 12,
+        APIC_MT_Startup, APIC_TGM_Edge, APIC_DSH_Destination);
+}

hal/halx86/apic/apictrap.S

@@ -32,6 +32,7 @@ TRAP_ENTRY HalpProfileInterrupt, KI_PUSH_FAKE_ERROR_CODE
 TRAP_ENTRY HalpTrap0D, 0
 TRAP_ENTRY HalpApcInterrupt, KI_PUSH_FAKE_ERROR_CODE
 TRAP_ENTRY HalpDispatchInterrupt, KI_PUSH_FAKE_ERROR_CODE
+TRAP_ENTRY HalpIpiInterrupt, KI_PUSH_FAKE_ERROR_CODE
 
 PUBLIC _ApicSpuriousService
 _ApicSpuriousService:

hal/halx86/apic/halinit.c

@@ -38,7 +38,30 @@ HalpInitProcessor(
 
     /* Initialize the local APIC for this cpu */
     ApicInitializeLocalApic(ProcessorNumber);
+    if(ProcessorNumber != 0)
+    {
+        ULONG_PTR EFlags;
 
+        /* Save EFlags and disable interrupts */
+        EFlags = __readeflags();
+        _disable();
+
+        /* Set interrupt handlers in the IDT */
+        KeRegisterInterruptHandler(APIC_CLOCK_VECTOR, HalpClockInterrupt);
+        KeRegisterInterruptHandler(APIC_IPI_VECTOR, HalpIpiInterrupt);
+    #ifndef _M_AMD64
+        KeRegisterInterruptHandler(APC_VECTOR, HalpApcInterrupt);
+        KeRegisterInterruptHandler(DISPATCH_VECTOR, HalpDispatchInterrupt);
+    #endif
+
+        /* Register the vectors for APC and dispatch interrupts */
+        HalpRegisterVector(IDT_INTERNAL, 0, APC_VECTOR, APC_LEVEL);
+        HalpRegisterVector(IDT_INTERNAL, 0, DISPATCH_VECTOR, DISPATCH_LEVEL);
+        HalpRegisterVector(IDT_INTERNAL, 0, APIC_IPI_VECTOR, IPI_LEVEL);
+        /* Restore interrupt state */
+        EFlags |= EFLAGS_INTERRUPT_MASK;
+        __writeeflags(EFlags);
+    }
     /* Initialize profiling data (but don't start it) */
     HalInitializeProfiling();
 
@@ -62,6 +85,14 @@ HalpInitPhase0(IN PLOADER_PARAMETER_BLOCK LoaderBlock)
                                CLOCK2_LEVEL,
                                HalpClockInterrupt,
                                Latched);
+
+    /* Enable IPI interrupt handler */
+    HalpEnableInterruptHandler(IDT_INTERNAL,
+                               0,
+                               APIC_IPI_VECTOR,
+                               IPI_LEVEL,
+                               HalpIpiInterruptHandler,
+                               Latched);
 }
 
 VOID

hal/halx86/apic/processor.c

@@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID)
     return TRUE;
 }
 
-/*
- * @implemented
- */
-BOOLEAN
-NTAPI
-HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock,
-                      IN PKPROCESSOR_STATE ProcessorState)
-{
-    /* Ready to start */
-    return FALSE;
-}
-
 /*
  * @implemented
  */
@@ -62,15 +50,4 @@ HalProcessorIdle(VOID)
     __halt();
 }
 
-/*
- * @implemented
- */
-VOID
-NTAPI
-HalRequestIpi(KAFFINITY TargetProcessors)
-{
-    UNIMPLEMENTED;
-    __debugbreak();
-}
-
 /* EOF */

hal/halx86/apicsmp.cmake

@@ -0,0 +1,17 @@
+
+list(APPEND HAL_APICSMP_ASM_SOURCE
+    apic/apictrap.S
+    apic/tsccal.S)
+
+list(APPEND HAL_APICSMP_SOURCE
+    apic/apic.c
+    apic/apictimer.c
+    apic/apicsmp.c
+    apic/halinit.c
+    apic/processor.c
+    apic/rtctimer.c
+    apic/tsc.c)
+
+add_asm_files(lib_hal_apicsmp_asm ${HAL_APICSMP_ASM_SOURCE})
+add_library(lib_hal_apicsmp OBJECT ${HAL_APIC_SOURCE} ${lib_hal_apicsmp_asm})
+add_dependencies(lib_hal_apicsmp asm)

hal/halx86/generic/halinit.c

@@ -152,9 +152,11 @@ HalInitSystem(IN ULONG BootPhase,
 
         /* Do some HAL-specific initialization */
         HalpInitPhase1();
-
-        /* Initialize Phase 1 of the x86 emulator */
-        HalInitializeBios(1, LoaderBlock);
+        if (KeNumberProcessors < 2)
+        {
+            /* Initialize Phase 1 of the x86 emulator */
+            HalInitializeBios(1, LoaderBlock);
+        }
     }
 
     /* All done, return */

hal/halx86/generic/pic.c

@@ -82,3 +82,4 @@ HalpInitializeLegacyPICs(VOID)
     /* Mask all interrupts */
     __outbyte(PIC2_DATA_PORT, 0xFF);
 }
+

hal/halx86/generic/trap.S

@@ -19,5 +19,6 @@ TRAP_ENTRY HalpTrap0D, 0
 TRAP_ENTRY HalpApcInterrupt, KI_SOFTWARE_TRAP
 TRAP_ENTRY HalpClockInterrupt, KI_PUSH_FAKE_ERROR_CODE
 TRAP_ENTRY HalpProfileInterrupt, KI_PUSH_FAKE_ERROR_CODE
+TRAP_ENTRY HalpIpiInterrupt, KI_PUSH_FAKE_ERROR_CODE
 
 END

hal/halx86/generic/up.c

@@ -0,0 +1,32 @@
+/*
+ * PROJECT:     ReactOS Kernel
+ * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
+ * PURPOSE:     Core source file for UP alternative functions
+ * COPYRIGHT:  Copyright 2021 Justin Miller <justinmiller100@gmail.com>
+ */
+
+/* INCLUDES ******************************************************************/
+
+#include <hal.h>
+#define NDEBUG
+#include <debug.h>
+
+/* FUNCTIONS *****************************************************************/
+
+VOID
+NTAPI
+HalRequestIpi(KAFFINITY TargetProcessors)
+{
+    /* This should never be called in UP mode */
+    __debugbreak();
+}
+
+BOOLEAN
+NTAPI
+HalStartNextProcessor(
+    IN PLOADER_PARAMETER_BLOCK LoaderBlock,
+    IN PKPROCESSOR_STATE ProcessorState)
+{
+    /* Always return false on UP systems */
+    return FALSE;
+}

hal/halx86/include/halp.h

@@ -211,6 +211,7 @@ HalpEnableInterruptHandler(IN UCHAR Flags,
 VOID NTAPI HalpInitializePICs(IN BOOLEAN EnableInterrupts);
 VOID __cdecl HalpApcInterrupt(VOID);
 VOID __cdecl HalpDispatchInterrupt(VOID);
+VOID __cdecl HalpIpiInterrupt(VOID);
 PHAL_SW_INTERRUPT_HANDLER __cdecl HalpDispatchInterrupt2(VOID);
 DECLSPEC_NORETURN VOID FASTCALL HalpApcInterrupt2ndEntry(IN PKTRAP_FRAME TrapFrame);
 DECLSPEC_NORETURN VOID FASTCALL HalpDispatchInterrupt2ndEntry(IN PKTRAP_FRAME TrapFrame);

hal/halx86/include/smp.h

@@ -1,7 +1,7 @@
 /*
  * PROJECT:     ReactOS Kernel
  * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
- * PURPOSE:     Header File for SMP support
+ * PURPOSE:     Public Header File for SMP
  * COPYRIGHT:   Copyright 2021 Justin Miller <justinmiller100@gmail.com>
  */
 
@@ -41,3 +41,16 @@ HalpSetupProcessorsTable(
 
 VOID
 HalpPrintApicTables(VOID);
+
+/* APIC specific functions inside apic/apicsmp.c */
+
+VOID
+ApicStartApplicationProcessor(ULONG NTProcessorNumber, PHYSICAL_ADDRESS StartupLoc);
+
+VOID
+NTAPI
+HalpRequestIpi(KAFFINITY TargetProcessors);
+
+VOID
+FASTCALL
+HalpIpiInterruptHandler(IN PKTRAP_FRAME TrapFrame);

hal/halx86/pic/pic.c

@@ -1137,6 +1137,15 @@ HalpApcInterrupt2ndEntry(IN PKTRAP_FRAME TrapFrame)
     _HalpApcInterruptHandler(TrapFrame);
 }
 
+#ifndef CONFIG_SMP
+VOID
+FASTCALL
+HalpIpiInterruptHandler(IN PKTRAP_FRAME TrapFrame)
+{
+    /* DO nothing */
+}
+#endif
+
 DECLSPEC_NORETURN
 VOID
 FASTCALL

hal/halx86/pic/processor.c

@@ -38,18 +38,6 @@ HalAllProcessorsStarted(VOID)
     return TRUE;
 }
 
-/*
- * @implemented
- */
-BOOLEAN
-NTAPI
-HalStartNextProcessor(IN PLOADER_PARAMETER_BLOCK LoaderBlock,
-                      IN PKPROCESSOR_STATE ProcessorState)
-{
-    /* Ready to start */
-    return FALSE;
-}
-
 /*
  * @implemented
  */
@@ -62,15 +50,4 @@ HalProcessorIdle(VOID)
     __halt();
 }
 
-/*
- * @implemented
- */
-VOID
-NTAPI
-HalRequestIpi(KAFFINITY TargetProcessors)
-{
-    /* Not implemented on UP */
-    __debugbreak();
-}
-
 /* EOF */

hal/halx86/smp.cmake

@@ -2,8 +2,15 @@
 list(APPEND HAL_SMP_SOURCE
     generic/buildtype.c
     generic/spinlock.c
-    smp/smp.c)
+    smp/smp.c
+    smp/ipi.c)
 
-add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE})
+if(ARCH STREQUAL "i386")
+    list(APPEND HAL_SMP_ASM_SOURCE
+        smp/i386/apentry.S)
+endif()
+
+add_asm_files(lib_hal_smp_asm ${HAL_SMP_ASM_SOURCE})
+add_library(lib_hal_smp OBJECT ${HAL_SMP_SOURCE} ${lib_hal_smp_asm})
 add_dependencies(lib_hal_smp bugcodes xdk)
 target_compile_definitions(lib_hal_smp PRIVATE CONFIG_SMP)

hal/halx86/smp/i386/apentry.S

@@ -0,0 +1,181 @@
+/*
+ * PROJECT:     ReactOS HAL
+ * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
+ * PURPOSE:     Application processor startup code for i386
+ * COPYRIGHT:   Copyright 2021 Justin Miller <justinmiller100@gmail.com>
+ *              Copyright 2021 Victor Perevertkin <victor.perevertkin@reactos.org>
+ */
+
+#include <asm.inc>
+#include <ks386.inc>
+
+#define ZERO_OFFSET(f) (f - _APEntry16)
+#define PS(f) (f - _APEntryCpuState)
+
+PUBLIC _APEntry16
+PUBLIC _APEntry16End
+PUBLIC _APEntryJump32Offset
+PUBLIC _APEntryJump32Segment
+PUBLIC _TempPageTableAddr
+PUBLIC _APEntryCpuState
+PUBLIC _APEntry32
+
+.code16
+_APEntry16:
+    cli
+
+    /* Load final descriptor tables */
+#ifdef _USE_ML
+    data32 lgdt fword ptr cs:[ZERO_OFFSET(PS_Gdtr)]
+    data32 lidt fword ptr cs:[ZERO_OFFSET(PS_Idtr)]
+#else
+    data32 lgdt cs:[ZERO_OFFSET(PS_Gdtr)]
+    data32 lidt cs:[ZERO_OFFSET(PS_Idtr)]
+#endif
+
+    /* Processor state base address */
+    mov ebx, cs:[ZERO_OFFSET(PS_SelfPtr)]
+    mov esi, cs:[ZERO_OFFSET(PS_SegDs)]
+
+    /* Load temp page table */
+    mov eax, cs:[ZERO_OFFSET(_TempPageTableAddr)]
+    mov cr3, eax
+
+    /* Enable paging and protected mode */
+    mov eax, cs:[ZERO_OFFSET(PS_Cr0)]
+    mov cr0, eax
+
+    /* Long jump, 32bit address */
+    .byte HEX(66)
+    .byte HEX(EA)
+_APEntryJump32Offset:
+    .long 0
+_APEntryJump32Segment:
+    .word 0
+
+_APEntry16End:
+
+.align 4
+_TempPageTableAddr:
+    .long 0
+// Processor state
+_APEntryCpuState:
+PS_SelfPtr:
+    .long 0 // self pointer (for use in protected mode)
+PS_Esp:
+    .long 0
+PS_Eip:
+    .long 0
+PS_Eflags:
+    .long 0
+PS_SegCs:
+    .long 0
+PS_SegDs:
+    .long 0
+PS_SegEs:
+    .long 0
+PS_SegSs:
+    .long 0
+PS_SegFs:
+    .long 0
+PS_SegGs:
+    .long 0
+// KSPECIAL_REGISTERS
+PS_Cr0:
+    .long 0
+PS_Cr2:
+    .long 0
+PS_Cr3:
+    .long 0
+PS_Cr4:
+    .long 0
+PS_KernelDr0:
+    .long 0
+PS_KernelDr1:
+    .long 0
+PS_KernelDr2:
+    .long 0
+PS_KernelDr3:
+    .long 0
+PS_KernelDr6:
+    .long 0
+PS_KernelDr7:
+    .long 0
+    .space 2
+PS_Gdtr:
+    .word 0
+    .long 0
+    .space 2
+PS_Idtr:
+    .word 0
+    .long 0
+PS_Tr:
+    .word 0
+PS_Ldtr:
+    .word 0
+    .space 4*6 // reserved
+.endcode16
+
+.code32
+_APEntry32:
+    cli
+
+    /* Load segment registers from ProcessorState values */
+    /* DS should be the first one */
+    mov ds, esi
+
+    mov eax, [ebx + PS(PS_SegEs)]
+    mov es, eax
+    mov eax, [ebx + PS(PS_SegSs)]
+    mov ss, eax
+    mov eax, [ebx + PS(PS_SegFs)]
+    mov fs, eax
+    mov eax, [ebx + PS(PS_SegGs)]
+    mov gs, eax
+
+    /* Write CR registers with ProcessorState values */
+    mov eax, [ebx + PS(PS_Cr3)]
+    mov cr3, eax
+    mov eax, [ebx + PS(PS_Cr4)]
+    mov cr4, eax
+
+    /* Load debug registers */
+    mov eax, [ebx + PS(PS_KernelDr0)]
+    mov dr0, eax
+    mov eax, [ebx + PS(PS_KernelDr1)]
+    mov dr1, eax
+    mov eax, [ebx + PS(PS_KernelDr2)]
+    mov dr2, eax
+    mov eax, [ebx + PS(PS_KernelDr3)]
+    mov dr3, eax
+    mov eax, [ebx + PS(PS_KernelDr6)]
+    mov dr6, eax
+    mov eax, [ebx + PS(PS_KernelDr7)]
+    mov dr7, eax
+
+    /* Load TSS */
+    ltr word ptr [ebx + PS(PS_Tr)]
+
+    /* Load AP Stack*/
+    mov esp, [ebx + PS(PS_Esp)]
+
+    /* Load Eip and push it as a "return" address */
+    mov eax, [ebx + PS(PS_Eip)]
+    push eax
+
+    /* Load flags */
+    mov eax, [ebx + PS(PS_Eflags)]
+    sahf
+
+    /* Set up all GP registers */
+    xor edi, edi
+    xor esi, esi
+    xor ebp, ebp
+    xor ebx, ebx
+    xor edx, edx
+    xor ecx, ecx
+    xor eax, eax
+
+    /* Jump into the kernel */
+    ret
+END

hal/halx86/smp/ipi.c

@@ -0,0 +1,22 @@
+/*
+ * PROJECT:     ReactOS Kernel
+ * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
+ * PURPOSE:     Source file for IPI management
+ * COPYRIGHT:   Copyright 2021 Justin Miller <justinmiller100@gmail.com>
+ */
+
+/* INCLUDES ******************************************************************/
+
+#include <hal.h>
+#include <smp.h>
+#define NDEBUG
+#include <debug.h>
+
+/* GLOBALS *******************************************************************/
+
+VOID
+NTAPI
+HalRequestIpi(KAFFINITY TargetProcessors)
+{
+    HalpRequestIpi(TargetProcessors);
+}

hal/halx86/smp/smp.c

@@ -3,6 +3,7 @@
  * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
  * PURPOSE:     Core source file for SMP management
  * COPYRIGHT:   Copyright 2021 Justin Miller <justinmiller100@gmail.com>
+ *              Copyright 2021 Victor Perevertkin <victor.perevertkin@reactos.org>
  */
 
 /* INCLUDES ******************************************************************/
@@ -15,6 +16,36 @@
 /* GLOBALS *******************************************************************/
 
 extern PPROCESSOR_IDENTITY HalpProcessorIdentity;
+extern PHYSICAL_ADDRESS HalpLowStubPhysicalAddress;
+extern PVOID HalpLowStub;
+
+// The data necessary for a boot (stored inside HalpLowStub)
+extern PVOID APEntry16;
+extern PVOID APEntry16End;
+extern PVOID APEntry32;
+extern PVOID APEntryJump32Offset;
+extern PVOID APEntryJump32Segment;
+extern PVOID TempPageTableAddr;
+extern PVOID APEntryCpuState;
+
+/* TODO: MaxAPCount should be assigned by a Multi APIC table */
+ULONG MaxAPCount = 2;
+ULONG StartedProcessorCount = 1;
+
+typedef struct _AP_ENTRY_CPU_STATE
+{
+    PVOID SelfPtr;
+    UINT32 Esp;
+    UINT32 Eip;
+    UINT32 Eflags;
+    UINT32 SegCs;
+    UINT32 SegDs;
+    UINT32 SegEs;
+    UINT32 SegSs;
+    UINT32 SegFs;
+    UINT32 SegGs;
+    KSPECIAL_REGISTERS SpecialRegisters; 
+} AP_ENTRY_CPU_STATE, *PAP_ENTRY_CPU_STATE;
 
 /* FUNCTIONS *****************************************************************/
 
@@ -31,3 +62,128 @@ HalpSetupProcessorsTable(
     CurrentPrcb = KeGetCurrentPrcb();
     HalpProcessorIdentity[NTProcessorNumber].ProcessorPrcb = CurrentPrcb;
 }
+
+static
+VOID
+HalpMapAddressFlat(
+    _Inout_ PMMPDE PageDirectory,
+    _In_ PVOID VirtAddress,
+    _In_ PVOID TargetVirtAddress)
+{
+    if (TargetVirtAddress == NULL)
+        TargetVirtAddress = VirtAddress;
+
+    PMMPDE currentPde;
+
+    currentPde = &PageDirectory[MiAddressToPdeOffset(TargetVirtAddress)];
+
+    // Allocate a Page Table if there is no one for this address
+    if (currentPde->u.Long == 0)
+    {
+        PMMPTE pageTable = ExAllocatePoolZero(NonPagedPool, PAGE_SIZE, TAG_HAL);
+        ASSERT(pageTable);
+
+        currentPde->u.Hard.PageFrameNumber = MmGetPhysicalAddress(pageTable).QuadPart >> PAGE_SHIFT;
+        currentPde->u.Hard.Valid = TRUE;
+        currentPde->u.Hard.Write = TRUE;
+    }
+    
+    // Map the Page Table so we can add our VirtAddress there (hack around I/O memory mapper for that)
+    PHYSICAL_ADDRESS b = {.QuadPart = (ULONG_PTR)currentPde->u.Hard.PageFrameNumber << PAGE_SHIFT};
+
+    PMMPTE pageTable = MmMapIoSpace(b, PAGE_SIZE, MmCached);
+
+    PMMPTE currentPte = &pageTable[MiAddressToPteOffset(TargetVirtAddress)];
+    currentPte->u.Hard.PageFrameNumber = MmGetPhysicalAddress(VirtAddress).QuadPart >> PAGE_SHIFT;
+    currentPte->u.Hard.Valid = TRUE;
+    currentPte->u.Hard.Write = TRUE;
+
+    MmUnmapIoSpace(pageTable, PAGE_SIZE);
+
+    DPRINT("Map %p -> %p, PDE %u PTE %u\n",
+           TargetVirtAddress,
+           (PVOID)MmGetPhysicalAddress(VirtAddress).LowPart,
+           MiAddressToPdeOffset(TargetVirtAddress),
+           MiAddressToPteOffset(TargetVirtAddress));
+}
+
+static
+PHYSICAL_ADDRESS
+HalpInitTempPageTable(
+    _In_ PKPROCESSOR_STATE ProcessorState)
+{
+    PMMPDE pageDirectory = ExAllocatePoolZero(NonPagedPool, PAGE_SIZE, TAG_HAL);
+    ASSERT(pageDirectory);
+
+    // Map the low stub
+    HalpMapAddressFlat(pageDirectory, HalpLowStub, (PVOID)(ULONG_PTR)HalpLowStubPhysicalAddress.QuadPart);
+    HalpMapAddressFlat(pageDirectory, HalpLowStub, NULL);
+
+    // Map 32bit mode entry point
+    HalpMapAddressFlat(pageDirectory, &APEntry32, NULL);
+
+    // Map GDT
+    HalpMapAddressFlat(pageDirectory, (PVOID)ProcessorState->SpecialRegisters.Gdtr.Base, NULL);
+
+    // Map IDT
+    HalpMapAddressFlat(pageDirectory, (PVOID)ProcessorState->SpecialRegisters.Idtr.Base, NULL);
+
+    // __debugbreak();
+
+    return MmGetPhysicalAddress(pageDirectory);
+}
+
+BOOLEAN
+NTAPI
+HalStartNextProcessor(
+    _In_ PLOADER_PARAMETER_BLOCK LoaderBlock,
+    _In_ PKPROCESSOR_STATE ProcessorState)
+{
+    if (MaxAPCount > StartedProcessorCount)
+    { 
+        // Initalize the temporary page table
+        // TODO: clean it up after an AP boots successfully
+        ULONG_PTR initialCr3 = HalpInitTempPageTable(ProcessorState).QuadPart;
+
+        // Put the bootstrap code into low memory
+        RtlCopyMemory(HalpLowStub, &APEntry16,  ((ULONG_PTR)&APEntry16End - (ULONG_PTR)&APEntry16));
+
+        // Set the data for 16bit entry code
+        PUINT32 APEntryJump32OffsetPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&APEntryJump32Offset - (ULONG_PTR)&APEntry16);
+        PUINT32 APEntryJump32SegmentPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&APEntryJump32Segment - (ULONG_PTR)&APEntry16);
+        PUINT32 TempPageTableAddrPtr = (PUINT32)((ULONG_PTR)HalpLowStub + (ULONG_PTR)&TempPageTableAddr - (ULONG_PTR)&APEntry16);
+
+        PVOID buf = &APEntry32;
+
+        RtlCopyMemory(APEntryJump32OffsetPtr, &buf, sizeof(buf));
+        RtlCopyMemory(APEntryJump32SegmentPtr, &ProcessorState->ContextFrame.SegCs, sizeof(UINT16));
+        RtlCopyMemory(TempPageTableAddrPtr, &initialCr3, sizeof(initialCr3));
+
+        // Write processor state stuff
+        PAP_ENTRY_CPU_STATE apCpuState = (PVOID)((ULONG_PTR)HalpLowStub + ((ULONG_PTR)&APEntryCpuState - (ULONG_PTR)&APEntry16));
+        *apCpuState = (AP_ENTRY_CPU_STATE){
+            .SelfPtr = apCpuState,
+            .Esp = ProcessorState->ContextFrame.Esp,
+            .Eip = ProcessorState->ContextFrame.Eip,
+            .Eflags = ProcessorState->ContextFrame.EFlags,
+            .SegCs = ProcessorState->ContextFrame.SegCs,
+            .SegDs = ProcessorState->ContextFrame.SegDs,
+            .SegEs = ProcessorState->ContextFrame.SegEs,
+            .SegSs = ProcessorState->ContextFrame.SegSs,
+            .SegFs = ProcessorState->ContextFrame.SegFs,
+            .SegGs = ProcessorState->ContextFrame.SegGs,
+            .SpecialRegisters = ProcessorState->SpecialRegisters,
+        };
+
+        ApicStartApplicationProcessor(StartedProcessorCount, HalpLowStubPhysicalAddress);
+
+        StartedProcessorCount++;
+
+        return TRUE;
+    }
+    else
+    {
+        return FALSE;
+    }
+}
+

hal/halx86/up.cmake

@@ -1,7 +1,8 @@
 
 list(APPEND HAL_UP_SOURCE
     generic/buildtype.c
-    generic/spinlock.c)
+    generic/spinlock.c
+    generic/up.c)
 
 add_library(lib_hal_up OBJECT ${HAL_UP_SOURCE})
 add_dependencies(lib_hal_up bugcodes xdk)

ntoskrnl/ex/init.c

@@ -1558,6 +1558,11 @@ Phase1InitializationDiscard(IN PVOID Context)
         KeBootTimeBias = 0;
     }
 
+#ifdef CONFIG_SMP
+    /* Start Application Processors */
+    KeStartAllProcessors();
+#endif
+
     /* Initialize all processors */
     if (!HalAllProcessorsStarted()) KeBugCheck(HAL1_INITIALIZATION_FAILED);
 

ntoskrnl/include/internal/amd64/ke.h

@@ -476,6 +476,13 @@ KiSetTrapContext(
     _In_ PCONTEXT Context,
     _In_ KPROCESSOR_MODE RequestorMode);
 
+VOID
+NTAPI
+KiInitializePcr(IN PKIPCR Pcr,
+                IN ULONG ProcessorNumber,
+                IN PKTHREAD IdleThread,
+                IN PVOID DpcStack);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

ntoskrnl/include/internal/i386/ke.h

@@ -399,6 +399,17 @@ KiRundownThread(IN PKTHREAD Thread)
 #endif
 }
 
+CODE_SEG("INIT")
+VOID
+NTAPI
+KiInitializePcr(IN ULONG ProcessorNumber,
+                IN PKIPCR Pcr,
+                IN PKIDTENTRY Idt,
+                IN PKGDTENTRY Gdt,
+                IN PKTSS Tss,
+                IN PKTHREAD IdleThread,
+                IN PVOID DpcStack);
+
 FORCEINLINE
 VOID
 Ke386SetGdtEntryBase(PKGDTENTRY GdtEntry, PVOID BaseAddress)

ntoskrnl/include/internal/ke.h

@@ -11,6 +11,11 @@ extern "C"
 
 /* INTERNAL KERNEL TYPES ****************************************************/
 
+/* Matches windbg expectation */
+#define IPI_FROZEN_RUNNING 0
+#define IPI_FROZEN_HALTED 2
+#define IPI_FROZEN_THAWING 3
+
 typedef struct _WOW64_PROCESS
 {
   PVOID Wow64;

ntoskrnl/kd64/kdapi.c

@@ -1152,6 +1152,27 @@ KdpWriteIoSpace(IN PDBGKD_MANIPULATE_STATE64 State,
                  &KdpContext);
 }
 
+KCONTINUE_STATUS
+NTAPI
+KeSwitchFrozenProcessor(IN USHORT ProcessorNumber)
+{
+    PKPRCB Prcb, TargetPrcb;
+    Prcb = KeGetCurrentPrcb();
+
+    if (ProcessorNumber <= KeNumberProcessors)
+    {
+        KdpDprintf("Processor Switch triggered Procesor: %d\n", ProcessorNumber);
+        TargetPrcb = KiProcessorBlock[ProcessorNumber];
+        InterlockedExchange((LONG*)&TargetPrcb->IpiFrozen, IPI_FROZEN_RUNNING);
+        InterlockedExchange((LONG*)&Prcb->IpiFrozen, IPI_FROZEN_HALTED);
+        return ContinueSuccess;
+    }
+    else
+    {
+        return ContinueProcessorReselected;
+    }
+}
+
 VOID
 NTAPI
 KdpReadIoSpaceExtended(IN PDBGKD_MANIPULATE_STATE64 State,
@@ -1267,6 +1288,7 @@ KdpSendWaitContinue(IN ULONG PacketType,
                     IN PSTRING SendData OPTIONAL,
                     IN OUT PCONTEXT Context)
 {
+    KCONTINUE_STATUS StatusCheck;
     STRING Data, Header;
     DBGKD_MANIPULATE_STATE64 ManipulateState;
     ULONG Length;
@@ -1469,11 +1491,10 @@ SendPacket:
                 break;
 
             case DbgKdSwitchProcessor:
-
-                /* TODO */
-                KdpDprintf("Processor Switch support is unimplemented!\n");
-                KdpNotSupported(&ManipulateState);
-                break;
+                KdRestore(FALSE);
+                StatusCheck = KeSwitchFrozenProcessor(ManipulateState.Processor);
+                KdSave(FALSE);
+                return StatusCheck;
 
             case DbgKdPageInApi:
 

ntoskrnl/ke/freeze.c

@@ -15,9 +15,50 @@
 
 /* GLOBALS ********************************************************************/
 
+extern KSPIN_LOCK KiFreezeExecutionLock;
+
 /* Freeze data */
 KIRQL KiOldIrql;
 ULONG KiFreezeFlag;
+VOID
+__cdecl
+KdpDprintf(
+    _In_ PCSTR Format,
+    ...);
+/* PRIVATE FUNCTIONS ***********************************************************/
+
+VOID
+NTAPI
+KiFreezeTargetExecution(_In_ PKTRAP_FRAME TrapFrame,
+                        _In_ PKEXCEPTION_FRAME ExceptionFrame)
+{
+    EXCEPTION_RECORD ExceptionRecord;
+    PKPRCB Prcb;
+    Prcb = KeGetCurrentPrcb();
+
+    if (TrapFrame)
+        KiSaveProcessorState(TrapFrame, ExceptionFrame);
+    /* Multiple processors can write this value */
+    InterlockedExchange((LONG*)&Prcb->IpiFrozen, IPI_FROZEN_HALTED);
+
+    /* Wait for triggering AP to give the go ahead to thaw */
+    while (Prcb->IpiFrozen != IPI_FROZEN_THAWING)
+    {
+        /* We only continue on if we are thawing, otherwise something else has happened! */
+        if (Prcb->IpiFrozen == IPI_FROZEN_RUNNING)
+        {
+            RtlZeroMemory(&ExceptionRecord, sizeof(EXCEPTION_RECORD));
+            ExceptionRecord.ExceptionAddress = (PVOID)Prcb->ProcessorState.ContextFrame.Eip;
+            KdpSwitchProcessor(&ExceptionRecord,
+                               (PCONTEXT)&Prcb->ProcessorState, FALSE);
+        }
+    }
+    //if (TrapFrame)
+        //KiRestoreProcessorControlState(TrapFrame, ExceptionFrame);
+    KeFlushCurrentTb();
+    /* Notify AP we're running once again */
+    InterlockedExchange((LONG*)&Prcb->IpiFrozen, IPI_FROZEN_RUNNING);
+}
 
 /* FUNCTIONS ******************************************************************/
 
@@ -26,6 +67,14 @@ NTAPI
 KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
                   IN PKEXCEPTION_FRAME ExceptionFrame)
 {
+#ifdef CONFIG_SMP
+    KAFFINITY TargetAffinity;
+    PKPRCB TargetPrcb;
+    KAFFINITY Current;
+    PKPRCB Prcb;
+    LONG i;
+#endif
+
     BOOLEAN Enable;
     KIRQL OldIrql;
 
@@ -49,7 +98,29 @@ KeFreezeExecution(IN PKTRAP_FRAME TrapFrame,
 #endif
 
 #ifdef CONFIG_SMP
-    // TODO: Add SMP support.
+    //TODO: add lock here
+
+    Prcb = KeGetCurrentPrcb();
+    TargetAffinity = KeActiveProcessors;
+    TargetAffinity &= ~Prcb->SetMember;
+    if (TargetAffinity)
+    {
+        KiIpiSend(TargetAffinity, IPI_FREEZE);
+        for (i = 0, Current = 1; i < KeNumberProcessors; i++, Current <<= 1)
+        {
+            if (TargetAffinity & Current)
+            {
+                /* stop target processor */
+                TargetPrcb = KiProcessorBlock[i];
+
+                /* Await for this processor to be frozen*/
+                while (TargetPrcb->IpiFrozen != IPI_FROZEN_HALTED)
+                {
+                    YieldProcessor();
+                }
+            }
+        }
+    }
 #endif
 
     /* Save the old IRQL to be restored on unfreeze */
@@ -64,7 +135,34 @@ NTAPI
 KeThawExecution(IN BOOLEAN Enable)
 {
 #ifdef CONFIG_SMP
-    // TODO: Add SMP support.
+    KAFFINITY TargetAffinity;
+    PKPRCB TargetPrcb;
+    KAFFINITY Current;
+    PKPRCB Prcb;
+    LONG i;
+
+    Prcb = KeGetCurrentPrcb();
+    TargetAffinity = KeActiveProcessors;
+    TargetAffinity &= ~Prcb->SetMember;
+
+    /* Loop through every processor */
+    for (i = 0, Current = 1; i < KeNumberProcessors; i++, Current <<= 1)
+    {
+        if (TargetAffinity & Current)
+        {
+            TargetPrcb = KiProcessorBlock[i];
+
+            /* Multiple processors can write this value */
+            InterlockedExchange((LONG*)&TargetPrcb->IpiFrozen, IPI_FROZEN_THAWING);
+
+            while (Prcb->IpiFrozen != IPI_FROZEN_RUNNING)
+            {
+                YieldProcessor();
+            }
+        }
+    }
+
+    //TODO: add unlock
 #endif
 
     /* Clear the freeze flag */

ntoskrnl/ke/i386/cpu.c

@@ -830,8 +830,12 @@ Ki386InitializeTss(IN PKTSS Tss,
     KiInitializeTSS2(Tss, TssEntry);
     KiInitializeTSS(Tss);
 
-    /* Load the task register */
-    Ke386SetTr(KGDT_TSS);
+    /* We only load the task register if this is BSP */
+    if(KeActiveProcessors == 0)
+    {
+        /* Load the task register */
+        Ke386SetTr(KGDT_TSS);
+    }
 
     /* Setup the Task Gate for Double Fault Traps */
     TaskGateEntry = (PKGDTENTRY)&Idt[8];

ntoskrnl/ke/i386/kiinit.c

@@ -536,7 +536,8 @@ KiInitializeKernel(IN PKPROCESS InitProcess,
     else
     {
         /* FIXME */
-        DPRINT1("SMP Boot support not yet present\n");
+        DPRINT1("Starting CPU#%u - you are brave!\n", Number);
+        KeLowerIrql(DISPATCH_LEVEL);
     }
 
     /* Setup the Idle Thread */
@@ -824,7 +825,11 @@ AppCpuInit:
     __writefsdword(KPCR_SET_MEMBER_COPY, 1 << Cpu);
     __writefsdword(KPCR_PRCB_SET_MEMBER, 1 << Cpu);
 
-    KiVerifyCpuFeatures(Pcr->Prcb);
+    if (!Cpu)
+    {
+        //TODO: This causes a bug check when ran on APs
+        KiVerifyCpuFeatures(Pcr->Prcb);
+    }
 
     /* Initialize the Processor with HAL */
     HalInitializeProcessor(Cpu, KeLoaderBlock);

ntoskrnl/ke/i386/mproc.c

@@ -0,0 +1,198 @@
+/*
+ * PROJECT:     ReactOS Kernel
+ * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
+ * PURPOSE:     Architecture specific source file to hold multiprocessor functions
+ * COPYRIGHT:   Copyright 2021 Justin Miller <justinmiller100@gmail.com>
+ *              Copyright 2021 Victor Perevertkin <victor.perevertkin@reactos.org>
+ */
+
+/* INCLUDES *****************************************************************/
+
+#include <ntoskrnl.h>
+// #define NDEBUG
+#include <debug.h>
+
+typedef struct _APINFO
+{
+    KIPCR Pcr;
+    ETHREAD Thread;
+    DECLSPEC_ALIGN(PAGE_SIZE) KGDTENTRY Gdt[128];
+    DECLSPEC_ALIGN(PAGE_SIZE) KIDTENTRY Idt[256];
+    KTSS Tss;
+    KTSS TssDoubleFault;
+    KTSS TssNMI;
+    DECLSPEC_ALIGN(16) UINT8 NMIStackData[DOUBLE_FAULT_STACK_SIZE];
+} APINFO, *PAPINFO;
+
+/* GLOBALS *******************************************************************/
+
+extern PLOADER_PARAMETER_BLOCK KeLoaderBlock;
+
+/* FUNCTIONS *****************************************************************/
+
+FORCEINLINE
+PKGDTENTRY
+KiGetGdtEntry(
+    IN PVOID pGdt,
+    IN USHORT Selector)
+{
+    return (PKGDTENTRY)((ULONG_PTR)pGdt + (Selector & ~RPL_MASK));
+}
+
+FORCEINLINE
+VOID
+KiSetGdtDescriptorBase(
+    IN OUT PKGDTENTRY Entry,
+    IN UINT32 Base)
+{
+    Entry->BaseLow = (UINT16)(Base & 0xffff);
+    Entry->HighWord.Bytes.BaseMid = (UINT8)((Base >> 16) & 0xff);
+    Entry->HighWord.Bytes.BaseHi  = (UINT8)((Base >> 24) & 0xff);
+}
+
+FORCEINLINE
+VOID
+KiSetGdtDescriptorLimit(
+    IN OUT PKGDTENTRY Entry,
+    IN UINT32 Limit)
+{
+    if (Limit < 0x100000)
+    {
+        Entry->HighWord.Bits.Granularity = 0;
+    }
+    else
+    {
+        Limit >>= 12;
+        Entry->HighWord.Bits.Granularity = 1;
+    }
+    Entry->LimitLow = (UINT16)(Limit & 0xffff);
+    Entry->HighWord.Bits.LimitHi = ((Limit >> 16) & 0x0f);
+}
+
+CODE_SEG("INIT")
+VOID
+NTAPI
+KeStartAllProcessors()
+{
+    PVOID KernelStack, DPCStack;
+    SIZE_T ProcessorCount = 0;
+    PAPINFO APInfo;
+
+    while (TRUE)
+    {
+        ProcessorCount++;
+        KernelStack = NULL;
+        DPCStack = NULL;
+
+        // Allocate structures for a new CPU.
+        APInfo = ExAllocatePoolZero(NonPagedPool, sizeof(APINFO), '  eK');
+        if (!APInfo)
+        {
+
+            break;
+        }
+        KernelStack = MmCreateKernelStack(FALSE, 0);
+        if (!KernelStack)
+        {
+            break;
+        }
+        DPCStack = MmCreateKernelStack(FALSE, 0);
+        if (!DPCStack)
+        {
+            break;
+        }
+
+        /* Initalize a new PCR for the specific AP */
+        KiInitializePcr(ProcessorCount,
+                        &APInfo->Pcr,
+                        &APInfo->Idt[0], 
+                        &APInfo->Gdt[0], 
+                        &APInfo->Tss,
+                        (PKTHREAD)&APInfo->Thread,
+                        DPCStack);
+
+        // Prepare descriptor tables
+        KDESCRIPTOR bspGdt, bspIdt;
+        __sgdt(&bspGdt.Limit);
+        __sidt(&bspIdt.Limit);
+        RtlCopyMemory(&APInfo->Gdt, (PVOID)bspGdt.Base, bspGdt.Limit + 1);
+        RtlCopyMemory(&APInfo->Idt, (PVOID)bspIdt.Base, bspIdt.Limit + 1);
+
+        KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_R0_PCR), (ULONG_PTR)&APInfo->Pcr);
+        KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_DF_TSS), (ULONG_PTR)&APInfo->TssDoubleFault);
+        KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_NMI_TSS), (ULONG_PTR)&APInfo->TssNMI);
+
+        KiSetGdtDescriptorBase(KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS), (ULONG_PTR)&APInfo->Tss);
+        // Clear TSS Busy flag (aka set the type to "TSS (Available)")
+        KiGetGdtEntry(&APInfo->Gdt, KGDT_TSS)->HighWord.Bits.Type = 0b1001;
+
+        APInfo->TssDoubleFault.Esp0 = (ULONG_PTR)&APInfo->NMIStackData;
+        APInfo->TssDoubleFault.Esp = (ULONG_PTR)&APInfo->NMIStackData;
+
+        APInfo->TssNMI.Esp0 = (ULONG_PTR)&APInfo->NMIStackData;
+        APInfo->TssNMI.Esp = (ULONG_PTR)&APInfo->NMIStackData;
+
+        // Push LOADER_BLOCK on stack as a parameter
+        KernelStack = (PVOID)((ULONG_PTR)KernelStack - sizeof(PVOID));
+        *(PVOID *)KernelStack = KeLoaderBlock;
+
+        // Push NULL address on stack as a "return" addr
+        KernelStack = (PVOID)((ULONG_PTR)KernelStack - sizeof(PVOID));
+        *(PVOID *)KernelStack = NULL;
+        
+        // Fill the processor state
+        PKPROCESSOR_STATE ProcessorState = &APInfo->Pcr.Prcb->ProcessorState;
+        RtlZeroMemory(ProcessorState, sizeof(*ProcessorState));
+
+        ProcessorState->SpecialRegisters.Cr0 = __readcr0();
+        ProcessorState->SpecialRegisters.Cr3 = __readcr3();
+        ProcessorState->SpecialRegisters.Cr4 = __readcr4();
+
+        ProcessorState->ContextFrame.SegCs = KGDT_R0_CODE;
+        ProcessorState->ContextFrame.SegDs = KGDT_R3_DATA;
+        ProcessorState->ContextFrame.SegEs = KGDT_R3_DATA;
+        ProcessorState->ContextFrame.SegSs = KGDT_R0_DATA;
+        ProcessorState->ContextFrame.SegFs = KGDT_R0_PCR;
+
+        ProcessorState->SpecialRegisters.Gdtr.Base = (ULONG_PTR)APInfo->Gdt;
+        ProcessorState->SpecialRegisters.Gdtr.Limit = sizeof(APInfo->Gdt) - 1;
+        ProcessorState->SpecialRegisters.Idtr.Base = (ULONG_PTR)APInfo->Idt;
+        ProcessorState->SpecialRegisters.Idtr.Limit = sizeof(APInfo->Idt) - 1;
+
+        ProcessorState->SpecialRegisters.Tr = KGDT_TSS;
+
+        ProcessorState->ContextFrame.Esp = (ULONG_PTR)KernelStack;
+        ProcessorState->ContextFrame.Eip = (ULONG_PTR)KiSystemStartup;
+        ProcessorState->ContextFrame.EFlags = __readeflags() & ~EFLAGS_INTERRUPT_MASK;
+
+        // Prepare the LOADER_PARAMETER_BLOCK structure
+        KeLoaderBlock->KernelStack = (ULONG_PTR)KernelStack;
+        KeLoaderBlock->Prcb = (ULONG_PTR)&APInfo->Pcr.Prcb;
+        KeLoaderBlock->Thread = (ULONG_PTR)&APInfo->Pcr.Prcb->IdleThread;
+
+        DPRINT("Starting CPU: #%u\n", ProcessorCount);
+        // Start the CPU
+        if (!HalStartNextProcessor(KeLoaderBlock, ProcessorState))
+        {
+            break;
+        }
+
+        while (READ_PORT_ULONG(&KeLoaderBlock->Prcb) != 0)
+        {
+            KeMemoryBarrier();
+            YieldProcessor();
+        }
+    }
+
+    // The last CPU didn't start - clean the data
+    ProcessorCount--;
+    
+    if (APInfo)
+        ExFreePoolWithTag(APInfo, '  eK');
+    if (KernelStack)
+        MmDeleteKernelStack(KernelStack, FALSE);
+    if (DPCStack)
+        MmDeleteKernelStack(DPCStack, FALSE);
+
+    DPRINT1("KeStartAllProcessors: Sucessful AP startup count is %u\n", ProcessorCount);
+}

ntoskrnl/ke/ipi.c

@@ -16,6 +16,12 @@
 
 extern KSPIN_LOCK KiReverseStallIpiLock;
 
+VOID
+NTAPI
+KiFreezeTargetExecution(_In_ PKTRAP_FRAME TrapFrame,
+                        _In_ PKEXCEPTION_FRAME ExceptionFrame);
+
+
 /* PRIVATE FUNCTIONS *********************************************************/
 
 VOID
@@ -29,15 +35,6 @@ KiIpiGenericCallTarget(IN PKIPI_CONTEXT PacketContext,
     ASSERTMSG("Not yet implemented\n", FALSE);
 }
 
-VOID
-FASTCALL
-KiIpiSend(IN KAFFINITY TargetProcessors,
-          IN ULONG IpiRequest)
-{
-    /* FIXME: TODO */
-    ASSERTMSG("Not yet implemented\n", FALSE);
-}
-
 VOID
 NTAPI
 KiIpiSendPacket(IN KAFFINITY TargetProcessors,
@@ -67,7 +64,6 @@ KiIpiSignalPacketDoneAndStall(IN PKIPI_CONTEXT PacketContext,
     ASSERTMSG("Not yet implemented\n", FALSE);
 }
 
-#if 0
 VOID
 NTAPI
 KiIpiSendRequest(IN KAFFINITY TargetSet,
@@ -85,13 +81,25 @@ KiIpiSendRequest(IN KAFFINITY TargetSet,
             /* Get the PRCB for this CPU */
             Prcb = KiProcessorBlock[i];
 
-            InterlockedBitTestAndSet((PLONG)&Prcb->IpiFrozen, IpiRequest);
-            HalRequestIpi(i);
+            InterlockedBitTestAndSet((PLONG)&Prcb->RequestSummary, IpiRequest);
         }
     }
+
+    /* HalRequestIpi does its own mask check =*/
+    HalRequestIpi(TargetSet);
 #endif
 }
 
+VOID
+FASTCALL
+KiIpiSend(IN KAFFINITY TargetProcessors, IN ULONG IpiRequest)
+{
+    /* Call private function */
+    KiIpiSendRequest(TargetProcessors, IpiRequest);
+}
+
+#if 0
+
 VOID
 NTAPI
 KiIpiSendPacket(IN KAFFINITY TargetSet,
@@ -145,47 +153,55 @@ KiIpiSendPacket(IN KAFFINITY TargetSet,
  */
 BOOLEAN
 NTAPI
-KiIpiServiceRoutine(IN PKTRAP_FRAME TrapFrame,
-                    IN PKEXCEPTION_FRAME ExceptionFrame)
+KiIpiServiceRoutine(IN PKTRAP_FRAME TrapFrame, IN PKEXCEPTION_FRAME ExceptionFrame)
 {
 #ifdef CONFIG_SMP
     PKPRCB Prcb;
     ASSERT(KeGetCurrentIrql() == IPI_LEVEL);
-
     Prcb = KeGetCurrentPrcb();
 
-    if (InterlockedBitTestAndReset((PLONG)&Prcb->IpiFrozen, IPI_APC))
+    /* APC level! Trigger an APC interrupt */
+    if (InterlockedBitTestAndReset((PLONG)&Prcb->RequestSummary, IPI_APC))
     {
         HalRequestSoftwareInterrupt(APC_LEVEL);
     }
 
-    if (InterlockedBitTestAndReset((PLONG)&Prcb->IpiFrozen, IPI_DPC))
+    /* DPC level! Trigger an DPC interrupt */
+    if (InterlockedBitTestAndReset((PLONG)&Prcb->RequestSummary, IPI_DPC))
     {
-        Prcb->DpcInterruptRequested = TRUE;
         HalRequestSoftwareInterrupt(DISPATCH_LEVEL);
     }
 
-    if (InterlockedBitTestAndReset((PLONG)&Prcb->IpiFrozen, IPI_SYNCH_REQUEST))
+    /* Freeze level! Trigger a FREEZE interrupt */
+    if (InterlockedBitTestAndReset((PLONG)&Prcb->RequestSummary, IPI_FREEZE))
+    {
+        KiFreezeTargetExecution(TrapFrame, ExceptionFrame);
+    }
+
+    /* SYNCH_REQUEST we have a function pointer to execute! */
+    if (InterlockedBitTestAndReset((PLONG)&Prcb->RequestSummary, IPI_SYNCH_REQUEST))
     {
 #if defined(_M_ARM) || defined(_M_AMD64)
-        DbgBreakPoint();
+            DbgBreakPoint();
 #else
-        (void)InterlockedDecrementUL(&Prcb->SignalDone->CurrentPacket[1]);
-        if (InterlockedCompareExchangeUL(&Prcb->SignalDone->CurrentPacket[2], 0, 0))
-        {
-            while (0 != InterlockedCompareExchangeUL(&Prcb->SignalDone->CurrentPacket[1], 0, 0));
-        }
-        ((VOID (NTAPI*)(PVOID))(Prcb->SignalDone->WorkerRoutine))(Prcb->SignalDone->CurrentPacket[0]);
-        InterlockedBitTestAndReset((PLONG)&Prcb->SignalDone->TargetSet, KeGetCurrentProcessorNumber());
-        if (InterlockedCompareExchangeUL(&Prcb->SignalDone->CurrentPacket[2], 0, 0))
-        {
-            while (0 != InterlockedCompareExchangeUL(&Prcb->SignalDone->TargetSet, 0, 0));
-        }
-        (void)InterlockedExchangePointer((PVOID*)&Prcb->SignalDone, NULL);
+            (void)InterlockedDecrementUL(&Prcb->CurrentPacket[1]);
+            if (InterlockedCompareExchangeUL(&Prcb->CurrentPacket[2], 0, 0))
+            {
+                while (0 != InterlockedCompareExchangeUL(&Prcb->CurrentPacket[1], 0, 0))
+                    ;
+            }
+            ((VOID(NTAPI *)(PVOID))(Prcb->WorkerRoutine))(Prcb->CurrentPacket[0]);
+            InterlockedBitTestAndReset((PLONG)&Prcb->TargetSet, KeGetCurrentProcessorNumber());
+            if (InterlockedCompareExchangeUL(&Prcb->CurrentPacket[2], 0, 0))
+            {
+                while (0 != InterlockedCompareExchangeUL(&Prcb->TargetSet, 0, 0))
+                    ;
+            }
+        (void)InterlockedExchangePointer((PVOID *)&Prcb->SignalDone, NULL);
 #endif // _M_ARM
     }
 #endif
-   return TRUE;
+    return TRUE;
 }
 
 /*
@@ -196,6 +212,8 @@ NTAPI
 KeIpiGenericCall(IN PKIPI_BROADCAST_WORKER Function,
                  IN ULONG_PTR Argument)
 {
+    /* TODO: merge rest of IPI code */
+    __debugbreak();
     ULONG_PTR Status;
     KIRQL OldIrql, OldIrql2;
 #ifdef CONFIG_SMP

ntoskrnl/ke/thrdobj.c

@@ -777,11 +777,16 @@ KeInitThread(IN OUT PKTHREAD Thread,
     PKTIMER Timer;
     NTSTATUS Status;
 
+   if (KeNumberProcessors >= 2)
+        __debugbreak();
+
     /* Initialize the Dispatcher Header */
     Thread->Header.Type = ThreadObject;
     Thread->Header.ThreadControlFlags = 0;
     Thread->Header.DebugActive = FALSE;
     Thread->Header.SignalState = 0;
+
+    /* CHECKPOINT - SMP seems to break when an AP runs this below, something else is CRITICALLY BROKEN!!!!!!*/
     InitializeListHead(&(Thread->Header.WaitListHead));
 
     /* Initialize the Mutant List */

ntoskrnl/ntos.cmake

@@ -333,6 +333,10 @@ if(ARCH STREQUAL "i386")
         ${REACTOS_SOURCE_DIR}/ntoskrnl/ps/i386/psldt.c
         ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmmain.c
         ${REACTOS_SOURCE_DIR}/ntoskrnl/vdm/vdmexec.c)
+    if(BUILD_MP)
+        list(APPEND SOURCE
+            ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/i386/mproc.c)
+    endif()
 elseif(ARCH STREQUAL "amd64")
     list(APPEND ASM_SOURCE
         ${REACTOS_SOURCE_DIR}/ntoskrnl/ke/amd64/boot.S

sdk/include/asm/ksx.template.h

@@ -285,7 +285,7 @@ HEADER("Stack sizes"),
 CONSTANT(KERNEL_STACK_SIZE), /// FIXME: Obsolete
 CONSTANT(KERNEL_LARGE_STACK_SIZE),
 CONSTANT(KERNEL_LARGE_STACK_COMMIT),
-//CONSTANT(DOUBLE_FAULT_STACK_SIZE),
+CONSTANT(DOUBLE_FAULT_STACK_SIZE),
 #ifdef _M_AMD64
 CONSTANT(KERNEL_MCA_EXCEPTION_STACK_SIZE),
 CONSTANT(NMI_STACK_SIZE),

sdk/include/ndk/kefuncs.h

@@ -73,6 +73,13 @@ KiDeliverApc(
 //
 // Process/Thread Functions
 //
+CODE_SEG("INIT")
+VOID
+NTAPI
+KeStartAllProcessors(
+    VOID
+);
+
 VOID
 NTAPI
 KeTerminateThread(

sdk/include/xdk/amd64/ke.h

@@ -184,6 +184,7 @@ $if (_NTDDK_)
 #define KERNEL_STACK_SIZE 0x6000
 #define KERNEL_LARGE_STACK_SIZE 0x12000
 #define KERNEL_LARGE_STACK_COMMIT KERNEL_STACK_SIZE
+#define DOUBLE_FAULT_STACK_SIZE 0x2000
 
 #define KERNEL_MCA_EXCEPTION_STACK_SIZE 0x2000
 

sdk/include/xdk/x86/ke.h

@@ -218,6 +218,7 @@ $if (_NTDDK_)
 #define KERNEL_STACK_SIZE                   12288
 #define KERNEL_LARGE_STACK_SIZE             61440
 #define KERNEL_LARGE_STACK_COMMIT           12288
+#define DOUBLE_FAULT_STACK_SIZE             0x3000
 
 #define SIZE_OF_80387_REGISTERS   80