From b65325770cef1e3dfc089007182731e6fb710297 Mon Sep 17 00:00:00 2001
From: magnum <magnum>
Date: Wed, 24 Aug 2011 01:47:24 +0200
Subject: [PATCH 35/39] Encoding support (upper/lower casing) for non-Unicode formats and Single mode

---
 src/DES_bs.c              |    6 +-
 src/NETLM_fmt_plug.c      |    3 +-
 src/NETLMv2_fmt_plug.c    |   52 ++--------------
 src/NETNTLMv2_fmt_plug.c  |   52 ++--------------
 src/NETSPLITLM_fmt_plug.c |    3 +-
 src/loader.c              |   33 +---------
 src/misc.c                |    2 +
 src/mscash1_fmt_plug.c    |    4 +-
 src/mscash2_fmt_plug.c    |    4 +-
 src/sapB_fmt_plug.c       |   19 ++----
 src/sapG_fmt_plug.c       |  149 +++++++++++----------------------------------
 src/single.c              |    5 +-
 src/unicode.c             |  139 ++++++++++++++++++++++++++++++++++++++----
 src/unicode.h             |   15 +++++
 14 files changed, 212 insertions(+), 274 deletions(-)

diff --git a/src/DES_bs.c b/src/DES_bs.c
index 89cf358..9c2bc0c 100644
--- a/src/DES_bs.c
+++ b/src/DES_bs.c
@@ -9,6 +9,7 @@
 #include "common.h"
 #include "DES_std.h"
 #include "DES_bs.h"
+#include "unicode.h"
 
 #if DES_BS_VECTOR
 #define DEPTH				[depth]
@@ -113,10 +114,7 @@ void DES_bs_init(int LM)
 
 	if (LM) {
 		for (c = 0; c < 0x100; c++)
-		if (c >= 'a' && c <= 'z')
-			DES_bs_all.E.extras.u[c] = c & ~0x20;
-		else
-			DES_bs_all.E.extras.u[c] = c;
+			DES_bs_all.E.extras.u[c] = CP_up[c];
 	}
 
 #if DES_BS_ASM
diff --git a/src/NETLM_fmt_plug.c b/src/NETLM_fmt_plug.c
index 1301b56..eb3c3e5 100644
--- a/src/NETLM_fmt_plug.c
+++ b/src/NETLM_fmt_plug.c
@@ -41,6 +41,7 @@
 #include "common.h"
 #include "formats.h"
 #include "memory.h"
+#include "unicode.h"
 
 #include <openssl/des.h>
 
@@ -273,7 +274,7 @@ static void netlm_set_key(char *key, int index)
 
 	/* Upper-case password */
 	for(; i<PLAINTEXT_LENGTH && saved_plain[index][i] != 0; i++)
-		if ((saved_plain[index][i] >= 'a') && (saved_plain[index][i] <= 'z')) saved_plain[index][i] ^= 0x20;
+		saved_plain[index][i] = CP_up[saved_plain[index][i]];
 
 	/* Generate 16-byte LM hash */
 	setup_des_key(saved_plain[index], &ks);
diff --git a/src/NETLMv2_fmt_plug.c b/src/NETLMv2_fmt_plug.c
index 74ef3bb..34df6a2 100644
--- a/src/NETLMv2_fmt_plug.c
+++ b/src/NETLMv2_fmt_plug.c
@@ -206,48 +206,6 @@ static int netlmv2_valid(char *ciphertext, struct fmt_main *pFmt)
 
 static char *netlmv2_prepare(char *split_fields[10], struct fmt_main *pFmt)
 {
-//	else if (options.format && ((strncmp(options.format, "netlmv2", 7)==0) ||
-//			(strncmp(options.format, "netntlmv2", 9)==0))) {
-//		char *srv_challenge = ldr_get_field(&line, db_options->field_sep_char);
-//		char *nethashv2 = ldr_get_field(&line, db_options->field_sep_char);
-//		char *cli_challenge = ldr_get_field(&line, db_options->field_sep_char);
-//		char *identity = NULL;
-//		int i;
-//
-//	/* DOMAIN\USER: -or- USER::DOMAIN: */
-//	if ((tmp = strstr(*login, "\\")) != NULL) {
-//		identity = (char *) mem_alloc(strlen(*login));
-//	  	strcpy(identity, tmp + 1);
-//
-//		/* Upper-Case Username - Not Domain */
-//		for(i=0; i<strlen(identity); i++)
-//		 if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
-//
-//		  strncat(identity, *login, tmp - *login);
-//	}
-//	else {
-//		identity = (char *) mem_alloc(strlen(*login) + strlen(uid) + 1);
-//		strcpy(identity, *login);
-//
-//			for(i=0; i<strlen(identity); i++)
-//		if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
-//
-//		strcat(identity, uid);
-//	}
-//	if (strncmp(options.format, "netlmv2", 7)==0) {
-//		  tmp = (char *) mem_alloc_tiny(9 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1, MEM_ALIGN_NONE);
-//		  sprintf(tmp, "$NETLMv2$%s$%s$%s$%s", identity, srv_challenge, nethashv2, cli_challenge);
-//		*ciphertext = tmp;
-//	}
-//	else {
-//		  tmp = (char *) mem_alloc_tiny(9 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1, MEM_ALIGN_NONE);
-//		  sprintf(tmp, "$NETNTLMv2$%s$%s$%s$%s", identity, srv_challenge, nethashv2, cli_challenge);
-//		  *ciphertext = tmp;
-//	}
-//
-//		MEM_FREE(identity);
-//	}
-
 	char *srv_challenge = split_fields[3];
 	char *nethashv2     = split_fields[4];
 	char *cli_challenge = split_fields[5];
@@ -261,20 +219,20 @@ static char *netlmv2_prepare(char *split_fields[10], struct fmt_main *pFmt)
 	/* DOMAIN\USER: -or- USER::DOMAIN: */
 	if ((tmp = strstr(login, "\\")) != NULL) {
 		identity = (char *) mem_alloc(strlen(login));
-	  	strcpy(identity, tmp + 1);
+		strcpy(identity, tmp + 1);
 
 		/* Upper-Case Username - Not Domain */
 		for(i=0; i<strlen(identity); i++)
-		 if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
+			identity[i] = CP_up[(unsigned char)identity[i]];
 
-		  strncat(identity, login, tmp - login);
+		strncat(identity, login, tmp - login);
 	}
 	else {
 		identity = (char *) mem_alloc(strlen(login) + strlen(uid) + 1);
 		strcpy(identity, login);
 
-			for(i=0; i<strlen(identity); i++)
-		if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
+		for(i=0; i<strlen(identity); i++)
+			identity[i] = CP_up[(unsigned char)identity[i]];
 		strcat(identity, uid);
 	}
 	tmp = (char *) mem_alloc(9 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1);
diff --git a/src/NETNTLMv2_fmt_plug.c b/src/NETNTLMv2_fmt_plug.c
index dba606f..cdf5ca7 100644
--- a/src/NETNTLMv2_fmt_plug.c
+++ b/src/NETNTLMv2_fmt_plug.c
@@ -212,48 +212,6 @@ static int netntlmv2_valid(char *ciphertext, struct fmt_main *pFmt)
 
 static char *netntlmv2_prepare(char *split_fields[10], struct fmt_main *pFmt)
 {
-//	else if (options.format && ((strncmp(options.format, "netlmv2", 7)==0) ||
-//			(strncmp(options.format, "netntlmv2", 9)==0))) {
-//		char *srv_challenge = ldr_get_field(&line, db_options->field_sep_char);
-//		char *nethashv2 = ldr_get_field(&line, db_options->field_sep_char);
-//		char *cli_challenge = ldr_get_field(&line, db_options->field_sep_char);
-//		char *identity = NULL;
-//		int i;
-//
-//	/* DOMAIN\USER: -or- USER::DOMAIN: */
-//	if ((tmp = strstr(*login, "\\")) != NULL) {
-//		identity = (char *) mem_alloc(strlen(*login));
-//	  	strcpy(identity, tmp + 1);
-//
-//		/* Upper-Case Username - Not Domain */
-//		for(i=0; i<strlen(identity); i++)
-//		 if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
-//
-//		  strncat(identity, *login, tmp - *login);
-//	}
-//	else {
-//		identity = (char *) mem_alloc(strlen(*login) + strlen(uid) + 1);
-//		strcpy(identity, *login);
-//
-//			for(i=0; i<strlen(identity); i++)
-//		if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
-//
-//		strcat(identity, uid);
-//	}
-//	if (strncmp(options.format, "netlmv2", 7)==0) {
-//		  tmp = (char *) mem_alloc_tiny(9 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1, MEM_ALIGN_NONE);
-//		  sprintf(tmp, "$NETLMv2$%s$%s$%s$%s", identity, srv_challenge, nethashv2, cli_challenge);
-//		*ciphertext = tmp;
-//	}
-//	else {
-//		  tmp = (char *) mem_alloc_tiny(9 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1, MEM_ALIGN_NONE);
-//		  sprintf(tmp, "$NETNTLMv2$%s$%s$%s$%s", identity, srv_challenge, nethashv2, cli_challenge);
-//		  *ciphertext = tmp;
-//	}
-//
-//		MEM_FREE(identity);
-//	}
-
 	char *srv_challenge = split_fields[3];
 	char *nethashv2     = split_fields[4];
 	char *cli_challenge = split_fields[5];
@@ -267,20 +225,20 @@ static char *netntlmv2_prepare(char *split_fields[10], struct fmt_main *pFmt)
 	/* DOMAIN\USER: -or- USER::DOMAIN: */
 	if ((tmp = strstr(login, "\\")) != NULL) {
 		identity = (char *) mem_alloc(strlen(login));
-	  	strcpy(identity, tmp + 1);
+		strcpy(identity, tmp + 1);
 
 		/* Upper-Case Username - Not Domain */
 		for(i=0; i<strlen(identity); i++)
-		 if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
+			identity[i] = CP_up[(unsigned char)identity[i]];
 
-		  strncat(identity, login, tmp - login);
+		strncat(identity, login, tmp - login);
 	}
 	else {
 		identity = (char *) mem_alloc(strlen(login) + strlen(uid) + 1);
 		strcpy(identity, login);
 
-			for(i=0; i<strlen(identity); i++)
-		if ((identity[i] >= 'a') && (identity[i] <= 'z')) identity[i] ^= 0x20;
+		for(i=0; i<strlen(identity); i++)
+			identity[i] = CP_up[(unsigned char)identity[i]];
 		strcat(identity, uid);
 	}
 	tmp = (char *) mem_alloc(11 + strlen(identity) + 1 + strlen(srv_challenge) + 1 + strlen(nethashv2) + 1 + strlen(cli_challenge) + 1);
diff --git a/src/NETSPLITLM_fmt_plug.c b/src/NETSPLITLM_fmt_plug.c
index 90c6f42..874a6cd 100644
--- a/src/NETSPLITLM_fmt_plug.c
+++ b/src/NETSPLITLM_fmt_plug.c
@@ -22,6 +22,7 @@
 #include "misc.h"
 #include "common.h"
 #include "formats.h"
+#include "unicode.h"
 
 #include <openssl/des.h>
 
@@ -228,7 +229,7 @@ static void nethalflm_set_key(char *key, int index)
 
 	/* Upper-case password */
 	for(; i<PLAINTEXT_LENGTH && saved_plain[index][i] != 0 ; i++)
-		if ((saved_plain[index][i] >= 'a') && (saved_plain[index][i] <= 'z')) saved_plain[index][i] ^= 0x20;
+		saved_plain[index][i] = CP_up[saved_plain[index][i]];
 
 	/* Generate first 8-bytes of LM hash */
 	setup_des_key(saved_plain[index], &ks);
diff --git a/src/loader.c b/src/loader.c
index eee5321..06d609a 100644
--- a/src/loader.c
+++ b/src/loader.c
@@ -28,6 +28,7 @@
 #ifdef HAVE_MPI
 #include "john-mpi.h"
 #endif
+#include "unicode.h"
 
 #ifdef HAVE_CRYPT
 extern struct fmt_main fmt_crypt;
@@ -39,15 +40,6 @@ extern struct fmt_main fmt_crypt;
 #define RF_ALLOW_MISSING		1
 #define RF_ALLOW_DIR			2
 
-/*
- * Word separator characters for ldr_split_words(), used on GECOS fields.
- */
-#define issep \
-	"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\177"
-
-static char issep_map[0x100];
-static int issep_initialized = 0;
-
 static char *no_username = "?";
 
 static void read_file(struct db_main *db, char *name, int flags,
@@ -81,21 +73,6 @@ static void read_file(struct db_main *db, char *name, int flags,
 	if (fclose(file)) pexit("fclose");
 }
 
-static void ldr_init_issep(void)
-{
-	char *pos;
-
-	if (issep_initialized) return;
-
-	memset(issep_map, 0, sizeof(issep_map));
-
-	memset(issep_map, 1, 33);
-	for (pos = issep; *pos; pos++)
-		issep_map[ARCH_INDEX(*pos)] = 1;
-
-	issep_initialized = 1;
-}
-
 void ldr_init_database(struct db_main *db, struct db_options *options)
 {
 	db->loaded = 0;
@@ -123,11 +100,9 @@ void ldr_init_database(struct db_main *db, struct db_options *options)
 
 		db->cracked_hash = NULL;
 
-		if (options->flags & DB_WORDS) {
+		if (options->flags & DB_WORDS)
 			options->flags |= DB_LOGIN;
 
-			ldr_init_issep();
-		}
 	}
 
 	list_init(&db->plaintexts);
@@ -397,11 +372,11 @@ static void ldr_split_string(struct list_main *dst, char *src)
 	pos = src;
 	do {
 		word = pos;
-		while (*word && issep_map[ARCH_INDEX(*word)]) word++;
+		while (*word && CP_isSeparator[ARCH_INDEX(*word)]) word++;
 		if (!*word) break;
 
 		pos = word;
-		while (!issep_map[ARCH_INDEX(*pos)]) pos++;
+		while (!CP_isSeparator[ARCH_INDEX(*pos)]) pos++;
 		c = *pos;
 		*pos = 0;
 		list_add_unique(dst, word);
diff --git a/src/misc.c b/src/misc.c
index 74c6f9e..7aaa1a5 100644
--- a/src/misc.c
+++ b/src/misc.c
@@ -148,6 +148,8 @@ char *strnzcat(char *dst, const char *src, int size)
 	return dst;
 }
 
+// NOTE there is an encoding-aware version in unicode.c: enc_strlwr(). That
+// one should be used for usernames, plaintexts etc in formats.
 #ifndef _MSC_VER
 char *strlwr(char *s)
 {
diff --git a/src/mscash1_fmt_plug.c b/src/mscash1_fmt_plug.c
index 9436167..4d978b4 100644
--- a/src/mscash1_fmt_plug.c
+++ b/src/mscash1_fmt_plug.c
@@ -161,8 +161,8 @@ static char * ms_split(char *ciphertext, int index)
 
 	out[i]=0;
 
-	// lowercase salt as well as hash
-	strlwr(&out[2]);
+	// lowercase salt as well as hash, encoding-aware
+	enc_strlwr(&out[2]);
 
 	return out;
 }
diff --git a/src/mscash2_fmt_plug.c b/src/mscash2_fmt_plug.c
index a7dcb97..4d3bc63 100644
--- a/src/mscash2_fmt_plug.c
+++ b/src/mscash2_fmt_plug.c
@@ -242,8 +242,8 @@ static char * ms_split(char *ciphertext, int index)
 
 	out[i] = 0;
 
-	// lowercase salt as well as hash
-	strlwr(&out[6]);
+	// lowercase salt as well as hash, encoding-aware
+	enc_strlwr(&out[6]);
 
 	return out;
 }
diff --git a/src/sapB_fmt_plug.c b/src/sapB_fmt_plug.c
index a486a93..070e0e2 100644
--- a/src/sapB_fmt_plug.c
+++ b/src/sapB_fmt_plug.c
@@ -6,6 +6,9 @@
  *
  * (c) x7d8 sap loverz, public domain, btw
  * cheers: see test-cases.
+ *
+ * sligthly modified by magnum 2011 to support the --encoding option
+ * No rights reserved.
  */
 
 /* char transition table for BCODE (from disp+work) */
@@ -42,6 +45,7 @@ unsigned char bcodeArr[BCODE_ARRAY_LENGTH]=
 #include "misc.h"
 #include "common.h"
 #include "formats.h"
+#include "unicode.h"
 
 #include "md5.h"
 
@@ -102,17 +106,6 @@ static void sapbcode_set_salt(void *salt)
 	memcpy(theSalt, salt, SALT_SIZE);
 }
 
-
-void strToUpper_B(char* str)
-{
-	int i=0;
-	if (NULL==str)
-		return;
-	for (i=0; i<strlen(str);i++)
-		if ((str[i] >= 'a') && (str[i] <= 'z')) str[i] ^= 0x20;
-}
-
-
 static void sapbcode_set_key(char *key, int index) {
 	strnzcpy(saved_key, key, PLAINTEXT_LENGTH+1);
 }
@@ -173,7 +166,7 @@ static void sapbcode_crypt_all(int count) {
 	strnzcpy(username, theSalt, calcActualSaltSize_B(theSalt)+1);
 	strnzcpy(password, saved_key, PLAINTEXT_LENGTH+1);
 
-  	strToUpper_B(password); //only UPPERCASE passwords accepted for BCODE
+  	enc_strupper(password); //only UPPERCASE passwords accepted for BCODE
 
 	//transform...
 	for (i=0; i<strlen(username); i++)
@@ -254,7 +247,7 @@ char *sapbcode_split(char *ciphertext, int index)
 	static char out[CIPHERTEXT_LENGTH + 1];
   	memset(out, 0, CIPHERTEXT_LENGTH + 1);
 	memcpy(out, ciphertext, CIPHERTEXT_LENGTH);
-	strToUpper_B(out); //username (==salt) && resulting hash can be uppercase...
+	enc_strupper(out); //username (==salt) && resulting hash can be uppercase...
 	return out;
 }
 
diff --git a/src/sapG_fmt_plug.c b/src/sapG_fmt_plug.c
index dd2ffd9..9466003 100644
--- a/src/sapG_fmt_plug.c
+++ b/src/sapG_fmt_plug.c
@@ -7,9 +7,8 @@
  * (c) x7d8 sap loverz, public domain, btw
  * cheers: see test-cases.
  *
- * sligthly modified by magnum 2011 to support the --encoding=utf8 flag (which will turn
- * off the incomplete iso-8859-1 conversion in translatePassword(), actually
- * boosting performance). No rights reserved.
+ * sligthly modified by magnum 2011 to support the --encoding option.
+ * No rights reserved.
  */
 
 //this array is from disp+work (sap's worker process)
@@ -36,6 +35,7 @@ unsigned char theMagicArray[MAGIC_ARRAY_SIZE]=
 #include "formats.h"
 #include "sha.h"
 #include "options.h"
+#include "unicode.h"
 
 #define FORMAT_LABEL			"sapg"
 #define FORMAT_NAME			"SAP CODVN G (PASSCODE)"
@@ -68,10 +68,13 @@ static struct fmt_tests sapcodvng_tests[] = {
 };
 
 static char saved_key[PLAINTEXT_LENGTH + 1];
+static UTF8 trPassword[PLAINTEXT_LENGTH + 1];
+static size_t pwLen;
 static ARCH_WORD_32 crypt_key[BINARY_SIZE / 4];
 static SHA_CTX ctx;
 
 static char theSalt[SALT_SIZE];
+static size_t unLen;
 
 static int sapcodvng_valid(char *ciphertext, struct fmt_main *pFmt)
 {
@@ -97,9 +100,27 @@ static int sapcodvng_valid(char *ciphertext, struct fmt_main *pFmt)
 	return 1;
 }
 
+/*
+ * this function is needed to determine the actual size of the salt (==username)
+ * theSalt has to point at the beginning of the actual salt. no more checks are done; relies on valid()
+ * this is needed because, afaik, john only supports salts w/ fixed length. sap uses the username, so we have to
+ * "strip" the padding (blanks at the end) for the calculation....
+ * usernames w/ spaces at the end are not supported (SAP does not support them either)
+ */
+int calcActualSaltSize_G(char* theSalt)
+{
+    int i;
+	if (NULL==theSalt)
+		return 0;
+	i=SALT_SIZE-1;
+	while (theSalt[i--]==0x20);
+	return SALT_SIZE-(SALT_SIZE-i)+2;
+}
+
 static void sapcodvng_set_salt(void *salt)
 {
 	memcpy(theSalt, salt, SALT_SIZE);
+	unLen=calcActualSaltSize_G(theSalt);
 }
 
 static void *sapcodvng_get_salt(char *ciphertext)
@@ -114,7 +135,16 @@ static void sapcodvng_init(struct fmt_main *pFmt)
 
 static void sapcodvng_set_key(char *key, int index)
 {
-	strnzcpy(saved_key, key, PLAINTEXT_LENGTH+1);
+	strnzcpy(saved_key, key, PLAINTEXT_LENGTH + 1);
+	// 0. translate candidates to UTF-8
+	if (options.utf8)
+		strnzcpy((char*)trPassword, key, PLAINTEXT_LENGTH + 1);
+	else { // convert from codepage -> Unicode -> UTF-8
+		UTF16 tmp16[PLAINTEXT_LENGTH + 1];
+		plaintowcs(tmp16, PLAINTEXT_LENGTH + 1, (UTF8*)key, PLAINTEXT_LENGTH);
+		utf16toutf8_r(trPassword, PLAINTEXT_LENGTH + 1, tmp16);
+	}
+	pwLen=strlen((char*)trPassword);
 }
 
 static char *sapcodvng_get_key(int index) {
@@ -134,24 +164,6 @@ static int sapcodvng_cmp_one(void * binary, int index)
 	return sapcodvng_cmp_all(binary, index);
 }
 
-
-/*
- * this function is needed to determine the actual size of the salt (==username)
- * theSalt has to point at the beginning of the actual salt. no more checks are done; relies on valid()
- * this is needed because, afaik, john only supports salts w/ fixed length. sap uses the username, so we have to
- * "strip" the padding (blanks at the end) for the calculation....
- * usernames w/ spaces at the end are not supported (SAP does not support them either)
- */
-int calcActualSaltSize_G(char* theSalt)
-{
-    int i;
-	if (NULL==theSalt)
-		return 0;
-	i=SALT_SIZE-1;
-	while (theSalt[i--]==0x20);
-	return SALT_SIZE-(SALT_SIZE-i)+2;
-}
-
 /*
  * calculate the length of data that has to be hashed from the magic array. pass the first hash result in here.
  * this is part of the walld0rf-magic
@@ -180,104 +192,18 @@ unsigned int extractOffsetToMagicArray(unsigned char *pbHashArray)
 	return modSum;
 }
 
-/*
- * used to alter the username which is always(!) uppercase...
- */
-void strToUpper_G(char* str)
-{
-	int i=0;
-	if (NULL==str)
-		return;
-	for (i=0; i<strlen(str);i++)
-		if ((str[i] >= 'a') && (str[i] <= 'z')) str[i] ^= 0x20;
-}
-
-/*
- * translation for passwords (chars >7bit ascii).
- * translates just some >7bit chars to some weird magic bytes (not unicode?!)
- * works for most west-european (e.g. french) passwords...
- *
- * Addendum by magnum:
- * The "weird magic" is actually UTF-8. The format uses UTF-8 internally so I
- * added the FMT_UTF8 to the format flags, and skip calling this ISO-8859-1
- * translation in case the --encoding=utf8 option is used.
- */
-static char* translatePassword(char* origPassword)
-{
-	static char password[PLAINTEXT_LENGTH*2];
-	unsigned int j=0, i;
-
-	for (i=0; i<strlen(origPassword); i++) {
-		if (origPassword[i] & 0x80) {
-			switch ((unsigned char)origPassword[i]) {
-				case 0xFC:
-					password[j]=0xC3; password[j+1]=0xBC; j+=2; break;
-				case 0xF6:
-					password[j]=0xC3; password[j+1]=0xB6; j+=2; break;
-				case 0xE4:
-					password[j]=0xC3; password[j+1]=0xA4; j+=2; break;
-				case 0xDC:
-					password[j]=0xC3; password[j+1]=0x9C; j+=2; break;
-				case 0xD6:
-					password[j]=0xC3; password[j+1]=0x96; j+=2; break;
-				case 0xC4:
-					password[j]=0xC3; password[j+1]=0x84; j+=2; break;
-				case 0xDF:
-					password[j]=0xC3; password[j+1]=0x9F; j+=2; break;
-				case 0xBA:
-					password[j]=0xC2; password[j+1]=0xB0; j+=2; break;
-				case 0xB4:
-					password[j]=0xC2; password[j+1]=0xB4; j+=2; break;
-				case 0xE9:
-					password[j]=0xC3; password[j+1]=0xA9; j+=2; break;
-				case 0xEA:
-					password[j]=0xC3; password[j+1]=0xAA; j+=2; break;
-				case 0xE8:
-					password[j]=0xC3; password[j+1]=0xA8; j+=2; break;
-				case 0xC9:
-					password[j]=0xC3; password[j+1]=0x89; j+=2; break;
-				case 0xCA:
-					password[j]=0xC3; password[j+1]=0x8A; j+=2; break;
-				case 0xC8:
-					password[j]=0xC3; password[j+1]=0x88; j+=2; break;
-				case 0xA7:
-					password[j]=0xC2; password[j+1]=0xA7; j+=2; break;
-				default:
-					password[j]=origPassword[i]; j++;
-					break;
-				} //switch
-			} else
-				password[j++]=origPassword[i];
-	}
-	password[j]='\0';
-	return password;
-}
-
 static void sapcodvng_crypt_all(int count) {
-	static char temp_key[BINARY_SIZE+1];
+	char temp_key[BINARY_SIZE+1];
 	unsigned int offsetMagicArray;
 	unsigned int lengthIntoMagicArray;
 	unsigned char secondHashData[PLAINTEXT_LENGTH*2+MAGIC_ARRAY_SIZE]; //max size...
 	size_t secondHashDataLen;
-	size_t pwLen, unLen;
 	char tempVar[PLAINTEXT_LENGTH*2]; 	//PLAINTEXT_LENGTH is maxlen of password AND maxlen of username
-	char* trPassword;
-
-	//0.	translate special characters in the password
-	if (options.utf8)
-		trPassword = saved_key;
-	else
-		trPassword=translatePassword(saved_key);
-
-	pwLen=strlen(trPassword);
-
-	unLen=calcActualSaltSize_G(theSalt);
 
 	//1.	we need to SHA1 the password and username
-	strnzcpy(tempVar, trPassword, PLAINTEXT_LENGTH);  //first: the password
+	strnzcpy(tempVar, (char*)trPassword, PLAINTEXT_LENGTH);  //first: the password
 	strnzcpy(tempVar+strlen(tempVar), theSalt, unLen+1); //second: the salt(username)
 
-
 	SHA1_Init(&ctx);
 	SHA1_Update(&ctx, (unsigned char*)tempVar, strlen(tempVar));
 	SHA1_Final((unsigned char*)&temp_key, &ctx);
@@ -294,7 +220,6 @@ static void sapcodvng_crypt_all(int count) {
 	SHA1_Init(&ctx);
 	SHA1_Update(&ctx, secondHashData, secondHashDataLen);
 	SHA1_Final((unsigned char*)crypt_key, &ctx);
-
 }
 
 static void * sapcodvng_binary(char *ciphertext)
@@ -369,7 +294,7 @@ char *sapcodvng_split(char *ciphertext, int index)
 	static char out[CIPHERTEXT_LENGTH + 1];
   	memset(out, 0, CIPHERTEXT_LENGTH + 1);
 	memcpy(out, ciphertext, CIPHERTEXT_LENGTH);
-	strToUpper_G(out); //username (==salt) && resulting hash can be uppercase...
+	enc_strupper(out); //username (==salt) && resulting hash can be uppercase...
 	return out;
 }
 
diff --git a/src/single.c b/src/single.c
index c5a34ee..ab70f53 100644
--- a/src/single.c
+++ b/src/single.c
@@ -23,6 +23,7 @@
 #ifdef HAVE_MPI
 #include "john-mpi.h"
 #endif
+#include "unicode.h"
 
 static int progress = 0;
 static int rec_rule;
@@ -290,7 +291,6 @@ static int single_process_pw(struct db_salt *salt, struct db_password *pw,
 	char pair[RULE_WORD_SIZE];
 	int split;
 	char *key;
-	unsigned int c;
 
 	if (!(first = pw->words->head))
 		return -1;
@@ -308,8 +308,7 @@ static int single_process_pw(struct db_salt *salt, struct db_password *pw,
 
 		if (++first_number > SINGLE_WORDS_PAIR_MAX) continue;
 
-		c = (unsigned int)first->data[0] | 0x20;
-		if (c < 'a' || c > 'z') continue;
+		if (!CP_isLetter[(unsigned char)first->data[0]]) continue;
 
 		second_number = 0;
 		second = pw->words->head;
diff --git a/src/unicode.c b/src/unicode.c
index 7ec55c0..1d72193 100644
--- a/src/unicode.c
+++ b/src/unicode.c
@@ -91,10 +91,16 @@ UTF16 utc2_downcase[0x10000];
  */
 UTF16 CP_to_Unicode[0x100];
 static UTF8  CP_from_Unicode[0x10000];
-static UTF8  CP_up[0x100];
-static UTF8  CP_down[0x100];
+UTF8  CP_up[0x100];
+UTF8  CP_down[0x100];
 static int UnicodeType = -1;
 
+/*
+ * This is used by single.c for determining that a character is a letter
+ */
+UTF8 CP_isLetter[0x100];
+UTF8 CP_isSeparator[0x100];
+
 #include "md4.h"
 #if !defined(uint16) && !defined(HAVE_UINT16_FROM_RPC_RPC_H)
 #if (SIZEOF_SHORT == 4)
@@ -588,6 +594,8 @@ UTF8 * utf16toplain_r (UTF8 *dst, int dst_len, const UTF16* source) {
 int initUnicode(int type) {
 	unsigned i;
 	unsigned char *cpU, *cpL;
+	char *encTemp;
+	char *pos;
 
 	if (UnicodeType==type) return 0;
 
@@ -714,8 +722,10 @@ int initUnicode(int type) {
 		cpU = (unsigned char*)CHARS_UPPER_CP858; cpL = (unsigned char*)CHARS_LOWER_CP858;
 	} else if (options.cp737) {
 		cpU = (unsigned char*)CHARS_UPPER_CP737; cpL = (unsigned char*)CHARS_LOWER_CP737;
-	} else {
+	} else if (options.iso8859_1) {
 		cpU = (unsigned char*)CHARS_UPPER_ISO_8859_1; cpL = (unsigned char*)CHARS_LOWER_ISO_8859_1;  // CHARS_LOW_ONLY_ISO_8859_1 is not needed.
+	} else {  // old-style ASCII-only
+		cpU = (unsigned char*)""; cpL = (unsigned char*)"";
 	}
 	for (i = 0; cpU[i]; ++i) {
 		CP_down[(unsigned)cpU[i]] = cpL[i];
@@ -736,6 +746,71 @@ int initUnicode(int type) {
 		CP_down[0x91] = 0x91;
 	}
 
+	memset(CP_isSeparator, 0, sizeof(CP_isSeparator));
+
+	// Original separator list from loader.c
+#define issep \
+	"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\177"
+
+	if (options.koi8_r)
+		encTemp = issep CHARS_PUNCTUATION_KOI8_R CHARS_SPECIALS_KOI8_R CHARS_WHITESPACE_KOI8_R CHARS_CONTROL_KOI8_R CHARS_INVALID_KOI8_R;
+	else if (options.cp1251)
+		encTemp = issep CHARS_PUNCTUATION_CP1251 CHARS_SPECIALS_CP1251 CHARS_WHITESPACE_CP1251 CHARS_CONTROL_CP1251 CHARS_INVALID_CP1251;
+	else if (options.cp866)
+		encTemp = issep CHARS_PUNCTUATION_CP866 CHARS_SPECIALS_CP866 CHARS_WHITESPACE_CP866 CHARS_CONTROL_CP866 CHARS_INVALID_CP866;
+	else if (options.cp1252)
+		encTemp = issep CHARS_PUNCTUATION_CP1252 CHARS_SPECIALS_CP1252 CHARS_WHITESPACE_CP1252 CHARS_CONTROL_CP1252 CHARS_INVALID_CP1252;
+	else if (options.iso8859_15)
+		encTemp = issep CHARS_PUNCTUATION_ISO_8859_15 CHARS_SPECIALS_ISO_8859_15 CHARS_WHITESPACE_ISO_8859_15 CHARS_CONTROL_ISO_8859_15 CHARS_INVALID_ISO_8859_15;
+	else if (options.cp437)
+		encTemp = issep CHARS_PUNCTUATION_CP437 CHARS_SPECIALS_CP437 CHARS_WHITESPACE_CP437 CHARS_CONTROL_CP437 CHARS_INVALID_CP437;
+	else if (options.cp850)
+		encTemp = issep CHARS_PUNCTUATION_CP850 CHARS_SPECIALS_CP850 CHARS_WHITESPACE_CP850 CHARS_CONTROL_CP850 CHARS_INVALID_CP850;
+	else if (options.cp858)
+		encTemp = issep CHARS_PUNCTUATION_CP858 CHARS_SPECIALS_CP858 CHARS_WHITESPACE_CP858 CHARS_CONTROL_CP858 CHARS_INVALID_CP858;
+	else if (options.cp737)
+		encTemp = issep CHARS_PUNCTUATION_CP737 CHARS_SPECIALS_CP737 CHARS_WHITESPACE_CP737 CHARS_CONTROL_CP737 CHARS_INVALID_CP737;
+	else if (options.iso8859_1)
+		encTemp = issep CHARS_PUNCTUATION_ISO_8859_1 CHARS_SPECIALS_ISO_8859_1 CHARS_WHITESPACE_ISO_8859_1 CHARS_CONTROL_ISO_8859_1 CHARS_INVALID_ISO_8859_1;
+	else  // old-style ASCII-only
+		encTemp = issep;
+
+	memset(CP_isSeparator, 1, 33);
+	for (pos = encTemp; *pos; pos++)
+		CP_isSeparator[ARCH_INDEX(*pos)] = 1;
+
+	memset(CP_isLetter, 0, sizeof(CP_isLetter));
+
+	if (options.koi8_r)
+		encTemp = CHARS_ALPHA_KOI8_R;
+	else if (options.cp1251)
+		encTemp = CHARS_ALPHA_CP1251;
+	else if (options.cp866)
+		encTemp = CHARS_ALPHA_CP866;
+	else if (options.cp1252)
+		encTemp = CHARS_ALPHA_CP1252;
+	else if (options.iso8859_15)
+		encTemp = CHARS_ALPHA_ISO_8859_15;
+	else if (options.cp437)
+		encTemp = CHARS_ALPHA_CP437;
+	else if (options.cp850)
+		encTemp = CHARS_ALPHA_CP850;
+	else if (options.cp858)
+		encTemp = CHARS_ALPHA_CP858;
+	else if (options.cp737)
+		encTemp = CHARS_ALPHA_CP737;
+	else if (options.iso8859_1)
+		encTemp = CHARS_ALPHA_ISO_8859_1;
+	else
+		encTemp = "";
+
+	for (i = 'a'; i <= 'z'; i++)
+		CP_isLetter[i] = 1;
+	for (i = 'A'; i <= 'Z'; i++)
+		CP_isLetter[i] = 1;
+	for (pos = encTemp; *pos; pos++)
+		CP_isLetter[ARCH_INDEX(*pos)] = 1;
+
 	return 0;
 }
 
@@ -845,17 +920,27 @@ int utf8_lc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len) {
 		*dst = 0;
 		return src_len;
 	}
-	// too big for our function.  Someone needs to write one for longer strings.
-	if (src_len > 512*3)
-		return 0;
 	utf16len = utf8towcs(tmp16, 512, src, src_len);
 	if (utf16len <= 0)
-		return 0;
+		goto lcFallback;
 	utf16len = utf16_lc(tmp16l, 512, tmp16, utf16len);
 	if (utf16len <= 0)
-		return 0;
+		goto lcFallback;
 	utf16toplain_r(dst, dst_len, tmp16l);
 	return strlen((char*)dst);
+
+	// Limp-home mode: If we failed doing the right thing (garbage data) we
+	// just do ASCII lc
+lcFallback:
+	if (dst_len <= src_len)
+		src_len = dst_len - 1;
+	for (i = 0; i < src_len; ++i)
+		if (*src >= 'A' && *src <= 'Z')
+			*dst++ = *src++ | 0x20;
+		else
+			*dst++ = *src++;
+	*dst = 0;
+	return src_len;
 }
 
 int utf8_uc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len) {
@@ -892,15 +977,43 @@ int utf8_uc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len) {
 		*dst = 0;
 		return len;
 	}
-	// too big for our function.  Someone needs to write one for longer strings.
-	if (src_len > 512*3)
-		return 0;
 	utf16len = utf8towcs(tmp16, 512, src, src_len);
 	if (utf16len <= 0)
-		return 0;
+		goto ucFallback;
 	utf16len = utf16_uc(tmp16u, 512, tmp16, utf16len);
 	if (utf16len <= 0)
-		return 0;
+		goto ucFallback;
 	utf16toplain_r(dst, dst_len, tmp16u);
 	return strlen((char*)dst);
+
+	// Limp-home mode: If we failed doing the right thing (garbage data) we
+	// just do ASCII uc
+ucFallback:
+	if (dst_len <= src_len)
+		src_len = dst_len - 1;
+	for (i = 0; i < src_len; ++i)
+		if (*src >= 'a' && *src <= 'z')
+			*dst++ = *src++ | 0x20;
+		else
+			*dst++ = *src++;
+	*dst = 0;
+	return src_len;
+}
+
+// Encoding-aware strlwr(): Simple in-place lowercasing
+char *enc_strlwr(char *s)
+{
+	unsigned char *ptr = (unsigned char *)s;
+	int srclen = strlen(s);
+	utf8_lc(ptr, srclen + 1, ptr, srclen);
+	return s;
+}
+
+// Simple in-place uppercasing
+char *enc_strupper(char *s)
+{
+	unsigned char *ptr = (unsigned char *)s;
+	int srclen = strlen(s);
+	utf8_uc(ptr, srclen + 1, ptr, srclen);
+	return s;
 }
diff --git a/src/unicode.h b/src/unicode.h
index 6c9383d..44eeabe 100644
--- a/src/unicode.h
+++ b/src/unicode.h
@@ -123,7 +123,22 @@ extern int utf16_uc(UTF16 *dst, unsigned dst_len, const UTF16 *src, unsigned src
 extern int utf8_lc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len);
 extern int utf8_uc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len);
 
+// Encoding-aware strlwr() (in-place lowercase)
+extern char *enc_strlwr(char *s);
+
+// Encoding-aware in-place uppercase
+extern char *enc_strupper(char *s);
+
 // Used by NT's inline set_key_helper_encoding()
 extern UTF16 CP_to_Unicode[0x100];
+
+// Used by various formats uc/lc
+extern UTF8  CP_up[0x100];
+extern UTF8  CP_down[0x100];
+
+// Used by single.c and loader.c
+extern UTF8 CP_isLetter[0x100];
+extern UTF8 CP_isSeparator[0x100];
+
 /* --------------------------------------------------------------------- */
 #endif				/* _CONVERTUTF_H */
-- 
1.7.4.1

