I'm not in the slightest bit confident about this, but assuming the big numbers posted are big endian, I wrote the following quick program on top of the arbitrary-precision arithmetic support in OpenSSL (whichever version came with Mac OS X v10.6):
#include <stdlib.h>
#include <openssl/bn.h>
#define BIGNUM_BIGENDIAN
#ifdef BIGNUM_BIGENDIAN
BIGNUM *load_number(unsigned char *buffer, int numbytes)
{
BIGNUM *r1 = BN_new();
BIGNUM *r2 = BN_new();
while(numbytes--)
{
BN_lshift(r2, r1, ;
BN_add_word(r2, *buffer); buffer++;
BIGNUM *t = r1; r1 = r2; r2 = t;
}
BN_free(r2);
return r1;
}
void print_number(BIGNUM *number)
{
BIGNUM *printCopy = BN_new(), *pc2 = BN_new();
BN_copy(printCopy, number);
unsigned char *tempBuffer;
tempBuffer = (unsigned char *)calloc(BN_num_bytes(number), 1);
int arrayPointer = BN_num_bytes(printCopy)-1;
while(!BN_is_zero(printCopy))
{
tempBuffer[arrayPointer] = BN_mod_word(printCopy, 256);
arrayPointer--;
BN_rshift(pc2, printCopy, ;
BIGNUM *t = pc2; pc2 = printCopy; printCopy = t;
}
BN_free(printCopy);
BN_free(pc2);
for(int c = 0; c < BN_num_bytes(number); c++)
{
printf("%02x, ", tempBuffer[c]);
}
printf("\n\n");
free(tempBuffer);
}
#endif
int main (int argc, const char * argv[]) {
static unsigned char keyfile_1[] = {
0xea, 0x6c, 0xad, 0xb2, 0xab, 0xb1, 0xd3, 0xee,
0x85, 0x6f, 0xd3, 0x36, 0xc0, 0xc1, 0x16, 0x1d,
0x31, 0x44, 0x65, 0x1a, 0x22, 0x81, 0xb5, 0xb8,
0x26, 0xdd, 0xce, 0x0f, 0x8f, 0xbb, 0x25, 0xc8,
0x1d, 0x34, 0x03, 0x1f, 0xb4, 0xb9, 0xae, 0xda,
0xcf, 0xde, 0x75, 0xc1, 0xd2, 0xed, 0x35, 0x4b,
0xcc, 0x11, 0x58
};
static unsigned char keyfile_2[] = {
0x14, 0xd6, 0x30, 0x08, 0x35, 0x57, 0x28, 0xef,
0x2b, 0xa3, 0x25, 0xb7, 0x11, 0x8c, 0x62, 0x2d,
0x16, 0x7a, 0x7d, 0xee, 0x57, 0xe7, 0x37, 0x18,
0xc9, 0x96, 0xe5, 0xa9, 0x63, 0x49, 0x68, 0x15,
0xf6, 0x6c, 0x12, 0x8c, 0x9e, 0xeb, 0xda, 0xef,
0xbd, 0x75, 0x3a, 0x9e, 0x7d, 0x02, 0xe6, 0xe9,
0xfd, 0xd7, 0x97
};
static unsigned char keyfile_3[] = {
0xdd, 0x74, 0xf0, 0xb7, 0xee, 0xe2, 0x6b, 0x6d,
0xb7, 0x75, 0xcc, 0xca, 0x1d, 0x65, 0xdc, 0xd4,
0x35, 0xe2, 0x09, 0xd0, 0x18, 0x46, 0x9b, 0xf5,
0x96, 0xcc, 0x80, 0xfa, 0x44, 0xea, 0xee, 0x0e,
0x23, 0xbb, 0x36, 0xfe, 0x68, 0x22, 0xbf, 0xb5,
0x53, 0x7d, 0xf2, 0xfb, 0x86, 0x82, 0x94, 0x13,
0xd4, 0x24, 0x6c
};
BIGNUM *keyFile1, *keyFile2, *keyFile3, *result;
keyFile1 = load_number(keyfile_1, 51);
keyFile2 = load_number(keyfile_2, 51);
keyFile3 = load_number(keyfile_3, 51);
BN_CTX *ctx = BN_CTX_new();
result = BN_new();
BN_mod_exp(result, keyFile1, keyFile2, keyFile3, ctx);
printf("Result is:\n");
print_number(result);
BN_free(result);
BN_free(keyFile1);
BN_free(keyFile2);
BN_free(keyFile3);
BN_CTX_free(ctx);
return 0;
}
I've never used OpenSSL before and couldn't find a definitive reference on the intended usage patterns for BIGNUMs, hence the slightly awkward means of byte stuffing and retrieval — though if you insert something like a print_number(keyFile1) then you get exactly the same byte pattern back as you put in, so I'm confident they're correct.
Anyway, output is:
90, 53, a4, ab, bd, 77, c6, b3, c6, 58, 5a, 41, 2d, d9, ce, 60, 47, 10, e3, 66, ab, 6b, 90, d8, 1e, ce, 3c, 05, de, 7b, 4f, 9f, 03, b7, 75, 12, 83, 2f, bd, 93, 81, 39, 1a, 41, 50, 34, b5, f8, 9c, 51, dc,
Obviously this will be completely incorrect if the 51 byte numbers given are intended to be little endian, in which case you'll need to modify load_number and print_number. Or just reverse the order of the numbers in the arrays then reverse the order of the numbers in the output.
Hope I've understood!