#!/usr/bin/env perl
# http://twiki.org/cgi-bin/view/Codev/ListingAllUndefinedButUsedWikiWords


$webDataLocation = "/usr/local/httpd/twiki/webs/Projects/data";
opendir(WEBDIR, $webDataLocation);
while($file=readdir(WEBDIR)) {
   next unless ($file =~ /\.txt$/);
   open(IN, "$webDataLocation/$file");
   $slurp= join (" ", <IN>);

   $slurp =~ s/\t/ /g;
   $slurp =~ s/\n/ /g;
   $slurp =~ s/\r/ /g;
   $slurp =~ s/[^a-zA-Z0-9 ]/ /g;
   $slurp =~ s/\s+/ /g;

   foreach $word (split(/\s+/, $slurp)) {
      if ($word =~ /^[A-Z]+[^A-Z]+[A-Z]+[^A-Z]+$/) {
         $seen{$word}++;
         $seenIn{$word}{$file}++;
      }
   }
   close IN;
}
foreach $word (keys %seen) {
    if ( -e "$webDataLocation/$word.txt") {
        push (@exists, "$seen{$word} : $word ref'd by : " . (join(" ", sort keys %{$seenIn{$word}})) . "\n");
    } else {
        push (@notexists , "$seen{$word} : $word ref'd by : " . (join(" ", sort keys %{$seenIn{$word}})) . "\n");
    }
}
$EXISTS = join ("", sort { $b <=> $a } @exists);
$NOTEXISTS = join ("", sort { $b <=> $a } @notexists);

foreach $word (keys %seen) {
    if ( -e "$webDataLocation/$word.txt") {
        push (@exists, "$seen{$word} : $word ref'd by : " . (join(" ", sort keys %{$seenIn{$word}})) . "\n");
    } else {
        push (@notexists , "$seen{$word} : $word _referenced by : " . (join(" ", sort keys %{$seenIn{$word}})) . "_\n \%BR\%");
    }
}
$EXISTS = join ("", sort { $b <=> $a } @exists);
$NOTEXISTS = join ("", sort { $b <=> $a } @notexists);
print <<REPORT;
---+ List of non-existing pages
This page automagically lists all pages which are referenced but undefined (i.e. which show with a '?' in TWiki. This list is refreshed every 30 minutes.

*NB:* Do not add pages that are people's names. Those pages will be created when he or she registers in TWiki (in TWiki.TWikiRegistration).

Use this page to discover:
   * which important pages are missing (top of the list)
   * spelling errors in page links (bottom of the list)

---++ Pages that are referenced but undefined
$NOTEXISTS
REPORT
