From 6dfbb264886444827f0065113c9b414fb302d42a Mon Sep 17 00:00:00 2001
From: magnum <magnum>
Date: Wed, 17 Aug 2011 11:12:55 +0200
Subject: [PATCH 27/27] Optionally report/store Unicode plaintexts as UTF-8 regardless of -enc option

---
 run/john.conf |   16 ++++++++++++++++
 src/cracker.c |   32 +++++++++++++++++++++++++++++---
 src/john.c    |   54 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 src/logger.c  |   24 ++++++++++++++++--------
 src/logger.h  |    2 +-
 src/options.h |    7 +++++++
 src/status.c  |   34 ++++++++++++++++++++++++++++------
 src/unicode.c |   20 ++++++++++++++++++++
 src/unicode.h |    6 ++++++
 9 files changed, 175 insertions(+), 20 deletions(-)

diff --git a/run/john.conf b/run/john.conf
index f2d12b9..6bab1ed 100644
--- a/run/john.conf
+++ b/run/john.conf
@@ -42,6 +42,22 @@ TimeFormat = %c
 # reported figures will be less accurate (default 0.05%)
 ETAthreshold = 0.05%
 
+# Always report (to screen and log) cracked passwords as UTF-8, regardless of
+# input encoding. This is recommended if you have your terminal set for UTF-8.
+AlwaysReportUTF8 = N
+
+# Always store Unicode (UTF-16) passwords as UTF-8 in john.pot, regardless
+# of input encoding. This prevents john.pot from being filled with mixed
+# and eventually unknown encodings. This is recommended if you have your
+# terminal set for UTF-8.
+UnicodeStoreUTF8 = N
+
+# Always report/store non-Unicode formats as UTF-8, regardless of input
+# encoding. There are pros and cons.
+CPstoreUTF8 = N
+
+# Write cracked passwords to the log file (default is just the user name)
+LogCrackedPasswords = N
 
 # A user defined character class is named with a single digit, ie. 0..9. After
 # the equal-sign, just list all characters that this class should match. You
diff --git a/src/cracker.c b/src/cracker.c
index a843284..07452df 100644
--- a/src/cracker.c
+++ b/src/cracker.c
@@ -19,6 +19,8 @@
 #include "logger.h"
 #include "status.h"
 #include "recovery.h"
+#include "options.h"
+#include "unicode.h"
 
 #ifdef index
 #undef index
@@ -92,16 +94,40 @@ void crk_init(struct db_main *db, void (*fix_state)(void),
 static int crk_process_guess(struct db_salt *salt, struct db_password *pw,
 	int index)
 {
+	UTF8 utf8buf[PLAINTEXT_BUFFER_SIZE + 1];
 	int dupe;
-	char *key;
+	char *key, *utf8key, *repkey;
 
 	dupe = !memcmp(&crk_timestamps[index], &status.crypts, sizeof(int64));
 	crk_timestamps[index] = status.crypts;
 
-	key = crk_methods.get_key(index);
+	repkey = key = crk_methods.get_key(index);
+
+	if (options.store_utf8 || options.report_utf8) {
+		if (options.utf8)
+			utf8key = key;
+		else {
+			utf8key = (char*)enctoutf8_r(key, utf8buf);
+#if 1
+			// Double-check that the conversion was correct. Our
+			// fallback is to log, warn and use the original key instead.
+			char tmp8[PLAINTEXT_BUFFER_SIZE + 1];
+			utf8toenc_r((UTF8*)utf8key, tmp8);
+			if (strcmp(tmp8, key)) {
+				fprintf(stderr, "Warning, conversion failed %s -> %s -> %s - fallback to codepage\n", key, utf8key, tmp8);
+				log_event("Warning, conversion failed %s -> %s -> %s - fallback to codepage", key, utf8key, tmp8);
+				utf8key = key;
+			}
+#endif
+		}
+		if (options.report_utf8)
+			repkey = utf8key;
+		if (options.store_utf8)
+			key = utf8key;
+	}
 
 	log_guess(crk_db->options->flags & DB_LOGIN ? pw->login : "?",
-		dupe ? NULL : pw->source, key, crk_db->options->field_sep_char);
+		dupe ? NULL : pw->source, repkey, key, crk_db->options->field_sep_char);
 
 	crk_db->guess_count++;
 	status.guess_count++;
diff --git a/src/john.c b/src/john.c
index b1edb27..8c8985c 100644
--- a/src/john.c
+++ b/src/john.c
@@ -194,8 +194,6 @@ static void john_log_format(void)
 			"tried in chunks of %d",
 			min_chunk > 1 ? "will" : "may",
 			chunk);
-	if (options.utf8 && database.format->params.flags & FMT_UTF8)
-		log_event("- UTF-8 conversion enabled");
 }
 
 static char *john_loaded_counts(void)
@@ -315,6 +313,25 @@ static void john_load(void)
 				database.format->params.algorithm_name);
 		}
 
+		options.log_passwords = cfg_get_bool(SECTION_OPTIONS,
+		    NULL, "LogCrackedPasswords", 0);
+		options.report_utf8 = cfg_get_bool(SECTION_OPTIONS,
+		    NULL, "AlwaysReportUTF8", 0);
+		if (database.format->params.flags & FMT_UNICODE)
+			options.store_utf8 = cfg_get_bool(SECTION_OPTIONS,
+			    NULL, "UnicodeStoreUTF8", 0);
+		else
+			options.store_utf8 = cfg_get_bool(SECTION_OPTIONS,
+			    NULL, "CPstoreUTF8", 0);
+
+		if (!options.utf8) {
+			if (options.report_utf8 && options.log_passwords)
+				log_event("- Passwords in this logfile are UTF-8 encoded");
+
+			if (options.store_utf8)
+				log_event("- Passwords will be stored UTF-8 encoded in .pot file");
+		}
+
 		total = database.password_count;
 		ldr_load_pot_file(&database, options.loader.activepot);
 		ldr_fix_database(&database);
@@ -405,6 +422,39 @@ static void john_init(char *name, int argc, char **argv)
 	sig_init();
 
 	john_load();
+
+	if (options.utf8)
+		log_event("- UTF-8 input encoding enabled");
+	else
+	if (options.iso8859_1)
+		log_event("- ISO-8859-1 input encoding enabled");
+	else
+	if (options.koi8_r)
+		log_event("- KOI8-R input encoding enabled");
+	else
+	if (options.cp1251)
+		log_event("- CP1251 input encoding enabled");
+	else
+	if (options.cp866)
+		log_event("- CP866 input encoding enabled");
+	else
+	if (options.cp1252)
+		log_event("- CP1252 input encoding enabled");
+	else
+	if (options.iso8859_15)
+		log_event("- ISO-8859-15 input encoding enabled");
+	else
+	if (options.cp437)
+		log_event("- CP437 input encoding enabled");
+	else
+	if (options.cp850)
+		log_event("- CP850 input encoding enabled");
+	else
+	if (options.cp858)
+		log_event("- CP858 input encoding enabled");
+	else
+	if (options.cp737)
+		log_event("- CP737 input encoding enabled");
 }
 
 static void john_run(void)
diff --git a/src/logger.c b/src/logger.c
index 08d8f1b..2af1dd4 100644
--- a/src/logger.c
+++ b/src/logger.c
@@ -42,6 +42,7 @@
 #endif
 
 static int cfg_beep;
+static int cfg_log_passwords;
 
 /*
  * Note: the file buffer is allocated as (size + LINE_BUFFER_SIZE) bytes
@@ -169,27 +170,30 @@ void log_init(char *log_name, char *pot_name, char *session)
 		cfg_beep = cfg_get_bool(SECTION_OPTIONS, NULL, "Beep", 0);
 	}
 
+	cfg_log_passwords = cfg_get_bool(SECTION_OPTIONS, NULL,
+	                                 "LogCrackedPasswords", 0);
+
 	in_logger = 0;
 }
 
-void log_guess(char *login, char *ciphertext, char *plaintext, char field_sep)
+void log_guess(char *login, char *ciphertext, char *rep_plain, char *store_plain, char field_sep)
 {
 	int count1, count2;
 
 #ifdef HAVE_MPI
-	// All but node 0 has stdout closed
+	// All but node 0 has stdout closed so we output to stderr
 	if (mpi_p > 1)
-		fprintf(stderr, "%-16s (%s)\n", plaintext, login);
+		fprintf(stderr, "%-16s (%s)\n", rep_plain, login);
 	else
 #endif
-	printf("%-16s (%s)\n", plaintext, login);
+	printf("%-16s (%s)\n", rep_plain, login);
 
 	in_logger = 1;
 
 	if (pot.fd >= 0 && ciphertext &&
-	    strlen(ciphertext) + strlen(plaintext) <= LINE_BUFFER_SIZE - 3) {
+	    strlen(ciphertext) + strlen(store_plain) <= LINE_BUFFER_SIZE - 3) {
 		count1 = (int)sprintf(pot.ptr,
-			"%s%c%s\n", ciphertext, field_sep, plaintext);
+			"%s%c%s\n", ciphertext, field_sep, store_plain);
 		if (count1 > 0) pot.ptr += count1;
 	}
 
@@ -198,8 +202,12 @@ void log_guess(char *login, char *ciphertext, char *plaintext, char field_sep)
 		count1 = log_time();
 		if (count1 > 0) {
 			log.ptr += count1;
-			count2 = (int)sprintf(log.ptr,
-				"+ Cracked %s\n", login);
+			if (cfg_log_passwords)
+				count2 = (int)sprintf(log.ptr,
+				    "+ Cracked %s: %s\n", login, rep_plain);
+			else
+				count2 = (int)sprintf(log.ptr,
+				    "+ Cracked %s\n", login);
 			if (count2 > 0)
 				log.ptr += count2;
 			else
diff --git a/src/logger.h b/src/logger.h
index 9e489b7..1cf1e73 100644
--- a/src/logger.h
+++ b/src/logger.h
@@ -21,7 +21,7 @@ extern void log_init(char *log_name, char *pot_name, char *session);
  * Prints a guessed password to stdout and logs it to john.pot (unless
  * ciphertext is NULL) and other related information to the log file.
  */
-extern void log_guess(char *login, char *ciphertext, char *plaintext, char field_sep);
+extern void log_guess(char *login, char *ciphertext, char *rep_plain, char *store_plain, char field_sep);
 
 /*
  * Logs an arbitrary event.
diff --git a/src/options.h b/src/options.h
index acc2a37..9aed47b 100644
--- a/src/options.h
+++ b/src/options.h
@@ -162,6 +162,13 @@ struct options_main {
 	int cp850;
 	int cp858;
 	int cp737;
+
+/* Show/log/store UTF-8 regardless of input decoding */
+	int store_utf8;
+	int report_utf8;
+
+/* Write cracked passwords to log (default is just username) */
+	int log_passwords;
 };
 
 extern struct options_main options;
diff --git a/src/status.c b/src/status.c
index c972e22..91079ca 100644
--- a/src/status.c
+++ b/src/status.c
@@ -37,6 +37,7 @@
 #include "status.h"
 #include "bench.h"
 #include "config.h"
+#include "unicode.h"
 
 #ifdef HAVE_MPI
 #include "logger.h"
@@ -439,11 +440,22 @@ static void status_print_cracking(char *percent)
 	if ((options.flags & FLG_STATUS_CHK) ||
 	    !(status.crypts.lo | status.crypts.hi))
 		trying[0] = 0;
-	else
+	else {
+		UTF8 t1buf[PLAINTEXT_BUFFER_SIZE + 1];
+		UTF8 t2buf[PLAINTEXT_BUFFER_SIZE + 1];
+		char *t1, *t2;
+		if (options.report_utf8 && !options.utf8) {
+			t1 = (char*)enctoutf8_r(crk_get_key1(), t1buf);
+			t2 = (char*)enctoutf8_r(saved_key, t2buf);
+		} else {
+			t1 = crk_get_key1();
+			t2 = saved_key;
+		}
 		snprintf(trying, sizeof(trying),
 		         "%strying: %s%s%s",
 		         mpi_p > 1 ? " " : "  ",
-		         crk_get_key1(), saved_key[0] ? " - " : "", saved_key);
+		         t1, t2[0] ? " - " : "", t2);
+	}
 
 	fprintf(stderr,
 	        "%s"
@@ -476,10 +488,20 @@ static void status_print_cracking(char *percent)
 	if ((options.flags & FLG_STATUS_CHK) ||
 	    !(status.crypts.lo | status.crypts.hi))
 		fputc('\n', stderr);
-	else
-		fprintf(stderr,
-			"  trying: %s%s%s\n",
-			crk_get_key1(), saved_key[0] ? " - " : "", saved_key);
+	else {
+		UTF8 t1buf[PLAINTEXT_BUFFER_SIZE + 1];
+		UTF8 t2buf[PLAINTEXT_BUFFER_SIZE + 1];
+		char *t1, *t2;
+		if (options.report_utf8 && !options.utf8) {
+			t1 = (char*)enctoutf8_r(crk_get_key1(), t1buf);
+			t2 = (char*)enctoutf8_r(saved_key, t2buf);
+		} else {
+			t1 = crk_get_key1();
+			t2 = saved_key;
+		}
+		fprintf(stderr,	"  trying: %s%s%s\n",
+		        t1, t2[0] ? " - " : "", t2);
+	}
 #endif
 }
 
diff --git a/src/unicode.c b/src/unicode.c
index 46401cf..6e907d7 100644
--- a/src/unicode.c
+++ b/src/unicode.c
@@ -541,6 +541,26 @@ UTF8 * utf16toutf8_r (UTF8 *dst, int dst_len, const UTF16* source) {
 	return dst;
 }
 
+/* Thread-safe conversion from codepage to UTF-8 */
+UTF8 * enctoutf8_r (char *src, UTF8* dst)
+{
+	UTF16 tmp16[PLAINTEXT_BUFFER_SIZE + 1];
+	plaintowcs(tmp16, PLAINTEXT_BUFFER_SIZE, (unsigned char*)src,
+	           PLAINTEXT_BUFFER_SIZE);
+	dst = utf16toutf8_r(dst, PLAINTEXT_BUFFER_SIZE, tmp16);
+	return dst;
+}
+
+/* Thread-safe conversion from UTF-8 to codepage */
+char * utf8toenc_r (UTF8 *src, char* dst)
+{
+	UTF16 tmp16[PLAINTEXT_BUFFER_SIZE + 1];
+	utf8towcs(tmp16, PLAINTEXT_BUFFER_SIZE, (unsigned char*)src,
+	           PLAINTEXT_BUFFER_SIZE);
+	dst = (char*)utf16toplain_r((UTF8*)dst, PLAINTEXT_BUFFER_SIZE, tmp16);
+	return dst;
+}
+
 // Convert UTF-16LE to UTF-8 or ISO-8859-1 depending on --encoding=utf8 flag.
 // This is not optimised as it's only used in get_key() as of now.
 // Non thread-safe version
diff --git a/src/unicode.h b/src/unicode.h
index 470715d..6c9383d 100644
--- a/src/unicode.h
+++ b/src/unicode.h
@@ -71,6 +71,12 @@ extern int utf8towcs(UTF16 * target, unsigned int maxtargetlen,
 extern int plaintowcs(UTF16 * dst, unsigned int maxdstlen, const UTF8 * src,
     unsigned int srclen);
 
+/* Thread-safe conversion from codepage to UTF-8 */
+UTF8 * enctoutf8_r (char *src, UTF8* dst);
+
+/* Thread-safe conversion from UTF-8 to codepage */
+char * utf8toenc_r (UTF8 *src, char* dst);
+
 /* Convert back to UTF-8 (for get_key without a saved_plain) */
 extern UTF8 * utf16toutf8 (const UTF16* source);
 extern UTF8 * utf16toutf8_r (UTF8 *dst, int dst_len, const UTF16* source);
-- 
1.7.4.1

