S12cybersecurity-YaraRules/RC5_Shellcode_Encryption.yar

rule RC5_Shellcode_Encryption
{
    meta:
        author = "0x12 Dark Development"
        description = "Detects usage of RC5 block encryption routine, likely for shellcode encryption"
        version = "1.0"

    strings:
        // Look for the sequence: A = rotate_left((A ^ B), B) + subkey
        $rol_xor_add_1 = { 8B ?? 33 ?? C1 C0 ?? 03 ?? }   // mov, xor, rol, add
        $rol_xor_add_2 = { 33 ?? C1 C0 ?? 03 ?? }         // xor, rol, add (shorter variant)
        
        // Look for use of rotate_left and XOR in loop structure
        $rc5_loop_signature = { 
            33 ??         // xor reg, reg
            C1 C0 ??      // rol reg, imm or reg
            03 ??         // add reg, reg/mem
            33 ??         // xor reg, reg
            C1 C0 ??      // rol reg, imm or reg
            03 ??         // add reg, reg/mem
        }

        // Optional: subkey setup pattern (S[0] = 0xb7e15163, S[i] += 0x9e3779b9)
        $key_schedule_constant = { 63 51 E1 B7 } // Little endian of 0xB7E15163
        $key_schedule_delta = { B9 77 37 9E }    // Little endian of 0x9E3779B9

    condition:
        (uint16(0) == 0x5A4D or uint32(0) == 0x464C457F) and
        (
            2 of ($rol_xor_add_*) or
            $rc5_loop_signature
        ) and
        1 of ($key_schedule_constant, $key_schedule_delta)
}