From c3e6a3fa4660dfc908b16a671773915843968f0d Mon Sep 17 00:00:00 2001
From: magnum <magnum>
Date: Mon, 22 Aug 2011 18:36:32 +0200
Subject: [PATCH 30/31] Fix for mkdumb16.pl and Dumb16/32 external modes

---
 run/john.conf          |   85 +++++++++++++++++++++++++++++++----------------
 src/unused/mkdumb16.pl |    2 +-
 2 files changed, 57 insertions(+), 30 deletions(-)
 mode change 100644 => 100755 src/unused/mkdumb16.pl

diff --git a/run/john.conf b/run/john.conf
index 6bab1ed..e72ffab 100644
--- a/run/john.conf
+++ b/run/john.conf
@@ -759,8 +759,8 @@ void restore()
 
 # Generic implementation of "dumb" exhaustive search of Unicode/UCS-2 and
 # an arbitrary charset. Default is to try *all* allocated characters (there's
-# 15801 of them). A fast format like NT can exhaust up to 3 characters in a
-# couple of weeks while four characters would take over 500 years...
+# 54473 of them). A fast format like NT can exhaust two characters in 15 minutes
+# while three characters would take 1,5 years...
 #
 # The output is UTF-8, so for 16-bit formats you need to give --enc=utf8
 [List.External:Dumb16]
@@ -768,7 +768,7 @@ int maxlength;            // Maximum password length to try
 int last;                 // Last character position, zero-based
 int lastid;               // Character index in the last position
 int id[0x7f];             // Current character indices for other positions
-int charset[0x4000], c0;  // Characters (raise to 0x10000 for *all* codepoints)
+int charset[0x10000], c0; // Characters
 int word16[0x7F];         // Word in UCS-2
 
 void init()
@@ -1536,10 +1536,16 @@ void init()
     while (c < 0x32ff)     // ..to CIRCLED KATAKANA WO
         charset[i++] = c++;
     c = 0x3300;            // from SQUARE APAATO
-    while (c < 0x3401)     // ..to <CJK Ideograph Extension A, First>
+    while (c < 0x3400)     // ..to SQUARE GAL
+        charset[i++] = c++;
+    c = 0x3400;            // from <CJK Ideograph Extension A, First>
+    while (c < 0x4db6)     // ..to <CJK Ideograph Extension A, Last>
         charset[i++] = c++;
     c = 0x4dc0;            // from HEXAGRAM FOR THE CREATIVE HEAVEN
-    while (c < 0x4e01)     // ..to <CJK Ideograph, First>
+    while (c < 0x4e00)     // ..to HEXAGRAM FOR BEFORE COMPLETION
+        charset[i++] = c++;
+    c = 0x4e00;            // from <CJK Ideograph, First>
+    while (c < 0x9fcc)     // ..to <CJK Ideograph, Last>
         charset[i++] = c++;
     c = 0xa000;            // from YI SYLLABLE IT
     while (c < 0xa48d)     // ..to YI SYLLABLE YYR
@@ -1638,6 +1644,9 @@ void init()
     c = 0xabf0;            // from MEETEI MAYEK DIGIT ZERO
     while (c < 0xabfa)     // ..to MEETEI MAYEK DIGIT NINE
         charset[i++] = c++;
+    c = 0xac00;            // from <Hangul Syllable, First>
+    while (c < 0xd7a4)     // ..to <Hangul Syllable, Last>
+        charset[i++] = c++;
     c = 0xd7b0;            // from HANGUL JUNGSEONG O-YEO
     while (c < 0xd7c7)     // ..to HANGUL JUNGSEONG ARAEA-E
         charset[i++] = c++;
@@ -1790,17 +1799,17 @@ void filter()
     i = -1; c = 0;
 
     while (word16[++i]) {
-        if (word16[i] < 0x80) {
-            word[c++] = word16[i];
+        if (word16[i] >= 0x0800) {
+            word[c++] = 0xe0 | (word16[i]>>12);
+            word[c++] = 0x80 | (word16[i]>>6 & 0x3f);
+            word[c++] = 0x80 | (word16[i] & 0x3f);
         }
-        else if (word16[i] < 0x0800) {
+        else if (word16[i] >= 0x80) {
             word[c++] = 0xc0 | (word16[i]>>6);
             word[c++] = 0x80 | (word16[i] & 0x3f);
         }
         else {
-            word[c++] = 0xe0 | (word16[i]>>12);
-            word[c++] = 0x80 | (word16[i]>>6 & 0x3f);
-            word[c++] = 0x80 | (word16[i] & 0x3f);
+            word[c++] = word16[i];
         }
     }
     word[c] = 0;
@@ -1808,8 +1817,8 @@ void filter()
 
 # Generic implementation of "dumb" exhaustive search of FULL Unicode and
 # an arbitrary charset. Default is to try *all* allocated characters (there's
-# 23296 of them). A fast format like NT can exhaust up to 3 characters in a
-# couple of weeks while four characters would take thousands of years...
+# 109070 of them). A fast format like NT can exhaust two characters in one
+# hour while three characters would take 12 years...
 #
 # The output is UTF-8, so for 16-bit formats you need to give --enc=utf8
 [List.External:Dumb32]
@@ -1817,7 +1826,7 @@ int maxlength;            // Maximum password length to try
 int last;                 // Last character position, zero-based
 int lastid;               // Character index in the last position
 int id[0x7f];             // Current character indices for other positions
-int charset[0x8000], c0;  // Characters (raise to 100000 for *all* codepoints)
+int charset[0x20000], c0; // Characters (raise to 100000 for *all* codepoints)
 int word32[0x7F];         // Word in UTF32
 
 void init()
@@ -2585,10 +2594,16 @@ void init()
     while (c < 0x32ff)     // ..to CIRCLED KATAKANA WO
         charset[i++] = c++;
     c = 0x3300;            // from SQUARE APAATO
-    while (c < 0x3401)     // ..to <CJK Ideograph Extension A, First>
+    while (c < 0x3400)     // ..to SQUARE GAL
+        charset[i++] = c++;
+    c = 0x3400;            // from <CJK Ideograph Extension A, First>
+    while (c < 0x4db6)     // ..to <CJK Ideograph Extension A, Last>
         charset[i++] = c++;
     c = 0x4dc0;            // from HEXAGRAM FOR THE CREATIVE HEAVEN
-    while (c < 0x4e01)     // ..to <CJK Ideograph, First>
+    while (c < 0x4e00)     // ..to HEXAGRAM FOR BEFORE COMPLETION
+        charset[i++] = c++;
+    c = 0x4e00;            // from <CJK Ideograph, First>
+    while (c < 0x9fcc)     // ..to <CJK Ideograph, Last>
         charset[i++] = c++;
     c = 0xa000;            // from YI SYLLABLE IT
     while (c < 0xa48d)     // ..to YI SYLLABLE YYR
@@ -2687,6 +2702,9 @@ void init()
     c = 0xabf0;            // from MEETEI MAYEK DIGIT ZERO
     while (c < 0xabfa)     // ..to MEETEI MAYEK DIGIT NINE
         charset[i++] = c++;
+    c = 0xac00;            // from <Hangul Syllable, First>
+    while (c < 0xd7a4)     // ..to <Hangul Syllable, Last>
+        charset[i++] = c++;
     c = 0xd7b0;            // from HANGUL JUNGSEONG O-YEO
     while (c < 0xd7c7)     // ..to HANGUL JUNGSEONG ARAEA-E
         charset[i++] = c++;
@@ -3107,6 +3125,15 @@ void init()
     c = 0x1f700;            // from ALCHEMICAL SYMBOL FOR QUINTESSENCE
     while (c < 0x1f774)     // ..to ALCHEMICAL SYMBOL FOR HALF OUNCE
         charset[i++] = c++;
+    c = 0x20000;            // from <CJK Ideograph Extension B, First>
+    while (c < 0x2a6d7)     // ..to <CJK Ideograph Extension B, Last>
+        charset[i++] = c++;
+    c = 0x2a700;            // from <CJK Ideograph Extension C, First>
+    while (c < 0x2b735)     // ..to <CJK Ideograph Extension C, Last>
+        charset[i++] = c++;
+    c = 0x2b740;            // from <CJK Ideograph Extension D, First>
+    while (c < 0x2b81e)     // ..to <CJK Ideograph Extension D, Last>
+        charset[i++] = c++;
     c = 0x2f800;            // from CJK COMPATIBILITY IDEOGRAPH-2F800
     while (c < 0x2fa1e)     // ..to CJK COMPATIBILITY IDEOGRAPH-2FA1D
         charset[i++] = c++;
@@ -3175,24 +3202,24 @@ void filter()
     i = -1; c = 0;
 
     while (word32[++i]) {
-        if (word32[i] < 0x80) {
-            word[c++] = word32[i];
-        }
-        else if (word32[i] < 0x0800) {
-            word[c++] = 0xc0 | (word32[i]>>6);
-            word[c++] = 0x80 | (word32[i] & 0x3f);
-        }
-        else if (word32[i] < 0x10000) {
-            word[c++] = 0xe0 | (word32[i]>>12);
-            word[c++] = 0x80 | (word32[i]>>6 & 0x3f);
-            word[c++] = 0x80 | (word32[i] & 0x3f);
-        }
-        else {
+        if (word32[i] >= 0x10000) {
             word[c++] = 0xf0 | (word32[i]>>18);
             word[c++] = 0x80 | (word32[i]>>12 & 0x3f);
             word[c++] = 0x80 | (word32[i]>>6 & 0x3f);
             word[c++] = 0x80 | (word32[i] & 0x3f);
         }
+        else if (word16[i] >= 0x0800) {
+            word[c++] = 0xe0 | (word16[i]>>12);
+            word[c++] = 0x80 | (word16[i]>>6 & 0x3f);
+            word[c++] = 0x80 | (word16[i] & 0x3f);
+        }
+        else if (word16[i] >= 0x80) {
+            word[c++] = 0xc0 | (word16[i]>>6);
+            word[c++] = 0x80 | (word16[i] & 0x3f);
+        }
+        else {
+            word[c++] = word16[i];
+        }
     }
     word[c] = 0;
 }
diff --git a/src/unused/mkdumb16.pl b/src/unused/mkdumb16.pl
old mode 100644
new mode 100755
index 8f0b36f..0ac9bd5
--- a/src/unused/mkdumb16.pl
+++ b/src/unused/mkdumb16.pl
@@ -15,7 +15,7 @@ while (<>) {
 }
 my $first = 0; my $last = 0; my $inrange = 0;
 foreach my $entry (@alloced) {
-	if ($entry == $last+1) {
+	if (!($desc[$entry] =~ /, First/) && $entry == $last+1 || $desc[$entry] =~ /, Last/) {
 		if (!$inrange) {
 			$first = $last;
 		}
-- 
1.7.4.1

