#!/usr/bin/perl
#
# Turn a wordlist into a cdb file.
#
# cdb is a format invented by Dan Bernstein for fast, constant databases.  The
# database is fixed during creation and cannot be changed without rebuilding
# it, and is optimized for very fast access.  This program takes as input a
# wordlist file (a set of words separated by newlines) and turns it into a cdb
# file with the words as keys and the constant "1" as a value.  The resulting
# database is suitable for fast existence lookups in the wordlist, such as for
# password dictionary checks.

require 5.006;
use strict;
use warnings;

use File::Basename qw(basename);
use Getopt::Long qw(GetOptions);

# The path to the cdb utility, used to create the final database.  By default,
# the user's PATH is searched for cdb.
my $CDB = 'cdb';

# print with error checking and an explicit file handle.
#
# $fh   - Output file handle
# @args - Remaining arguments to print
#
# Returns: undef
#  Throws: Text exception on output failure
sub print_fh {
    my ($fh, @args) = @_;
    print {$fh} @args or croak('print failed');
    return;
}

# Always flush output.
STDOUT->autoflush;

# Clean up the script name for error reporting.
my $fullpath = $0;
local $0 = basename($0);

# Parse the argument list.
my ($ascii, $min_length, $manual);
Getopt::Long::config('bundling', 'no_ignore_case');
GetOptions(
    'ascii|a'        => \$ascii,
    'min-length|l=i' => \$min_length,
    'manual|man|m'   => \$manual
);
if ($manual) {
    print_fh(\*STDOUT, "Feeding myself to perldoc, please wait...\n");
    exec('perldoc', '-t', $fullpath);
}
if (@ARGV != 1) {
    die "Usage: cdbmake-wordlist <wordlist>\n";
}
my $input = $ARGV[0];

# The output file goes in the same directory and is named the same as the
# input but with .data appended.
my $output = $input . '.data';
if (-f $output) {
    die "$0: temporary output file $output already exists\n";
}

# Process the input file into the output file, converting it to cdb input
# format.
open(my $in, '<', $input)
  or die "$0: cannot open input file $input: $!\n";
open(my $out, '>', $output)
  or die "$0: cannot create output file $output: $!\n";
while (defined(my $word = <$in>)) {
    chomp($word);
    my $length = length($word);
    next if (defined($min_length) && $length < $min_length);
    if ($ascii) {
        next if $word =~ m{ [^[:ascii:]] }xms;
        next if $word =~ m{ [[:cntrl:]] }xms;
    }
    print_fh($out, "+$length,1:$word->1\n");
}
print_fh($out, "\n");
close($in)  or die "$0: cannot read all of input file $input: $!\n";
close($out) or die "$0: cannot write to output file $output: $!\n";

# Run cdb to turn the result into a constant database.  Ignore duplicate keys.
system($CDB, '-c', '-u', "$input.cdb", $output) == 0
  or die "$0: cdb -c failed\n";

# Remove the temporary file.
unlink($output) or die "$0: cannot remove temporary file $output: $!\n";

# All done.
exit(0);
__END__

=for stopwords
cdbmake-wordlist cdb whitespace wordlist lookups lookup sublicense
MERCHANTABILITY NONINFRINGEMENT krb5-strength --ascii Allbery

=head1 NAME

cdbmake-wordlist - Create a cdb database from a wordlist

=head1 SYNOPSIS

B<cdbmake-wordlist> [B<-am>] [B<-l> I<length>] I<wordlist>

=head1 DESCRIPTION

cdb is a format invented by Dan Bernstein for fast, constant databases.
The database is fixed during creation and cannot be changed without
rebuilding it, and is optimized for very fast access.  This program takes
as input a wordlist file (a set of words, possibly including whitespace,
separated by newlines) and turns it into a cdb file with the words as keys
and the constant C<1> as a value.  The resulting database is suitable for
fast existence lookups in the wordlist, such as for password dictionary
checks.

B<cdbmake-wordlist> takes one argument, the input wordlist file.  The
output cdb database will have the same name as I<wordlist> but with
C<.cdb> appended.  The input wordlist file does not have to be sorted.

=head1 OPTIONS

=over 4

=item B<-a>, B<--ascii>

Filter all words that contain non-ASCII characters or control characters
from the resulting cdb file, leaving only words that consist solely of
ASCII non-control characters.

=item B<-l> I<minimum>, B<--min-length>=I<minimum>

Filter all words of length less than I<minimum> from the resulting cdb
database.  The length of each line (minus the separating newline) in the
input wordlist will be checked against I<minimum> and will be filtered out
of the resulting database if it is shorter.  Useful for generating password
dictionaries where shorter passwords will be rejected by a generic length
check and no dictionary lookup will be done for a transform of the password
shorter than the specified minimum.

The default is not to filter out any words for minimum length.

=item B<-m>, B<--man>, B<--manual>

Print out this documentation (which is done simply by feeding the script to
C<perldoc -t>).

=back

=head1 AUTHOR

Russ Allbery <rra@stanford.edu>

=head1 COPYRIGHT AND LICENSE

Copyright 2013 The Board of Trustees of the Leland Stanford Junior
University

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

=head1 SEE ALSO

cdb(1)

The cdb file format is defined at L<http://cr.yp.to/cdb.html>.

The current version of this program is available from its web page at
L<http://www.eyrie.org/~eagle/software/krb5-strength/> as part of the
krb5-strength package.

=cut
