#! /usr/bin/perl -w
########################################################################

use strict;
use utf8;
use locale;
use POSIX qw(locale_h);
use POSIX qw(strftime);
use Encode 'from_to','encode_utf8','decode_utf8';
use English;
use Getopt::Long;
use File::Path;
use XML::Parser;
use Data::Dumper;

binmode STDIN,  ":utf8";
binmode STDOUT, ":utf8";
binmode STDERR, ":utf8";

#----------------------------------------------------------------------
# global variables

my $version = "0.20100824";

my $run_date = decode_utf8(`date +%Y-%m-%d`);
chomp ($run_date);

my $original_wd;
chomp ($original_wd = `pwd`);

my @handoffFiles = ();
my %handoffXmlTrees = ();

my %idsFromHandoffFiles = ();

my %idsFromSpecUsage = ();
my $totalNumberOfIdsFoundInSpec = 0;
my $totalNumberOfIdsFoundInSpecUsed = 0;
my $totalNumberOfIdsFoundInSpecButNeverUsed = 0;

my %idsFoundInEePackages = ();
my $totalNumberOfIdsFoundInEePackages = 0;
my $totalNumberOfIdsFoundInEePackagesButNotInSpec = 0;
my $totalNumberOfIdsFoundInEePackagesLackingEngineeringEnglish = 0;

my %idsFromSourcePackages = ();
my $totalNumberOfIdsFromSourcePackages = 0;
my $totalNumberOfIdsFoundInSourcePackagesButNotInSpec = 0;
my $totalNumberOfIdsFoundInSourcePackagesLackingEngineeringEnglish = 0;

my $htmlOutIdsFromEePackagesSummaryTable = "";
my $htmlOutIdsFromEePackagesDetail = "";

my $htmlOutIdsFromSourcePackagesSummaryTable = "";
my $htmlOutIdsFromSourcePackagesDetail = "";

my $csvOut = "Invalid Message ID, Engineering English .qm file, Package, Package version, Package maintainer\n";

my $class;

my $OPT_VERBOSITY;
my $OPT_VERSION;
my $OPT_HELP;
my $OPT_EEDIR;
my $OPT_HANDOFF_URL;
my $OPT_HANDOFF_DIR;
my $OPT_SOURCEPACKAGES_DIR;
my $OPT_HTTP_USER;
my $OPT_HTTP_PASSWD;
my $OPT_SKIP_DOWNLOAD;
my $OPT_OUTPUTDIR;

# Engineering English packages which should be ignored because they are obsolete:
my @engineeringEnglishPackagesObsolete = (
    "libdui-l10n-engineering-english", # now "libmeegotouch-l10n-engineering-english"
    "duistartup-l10n-engineering-english", # now "mstartup-l10n-engineering-english"
    "dui-demos-widgetsgallery-l10n-engineering-english", # now "meegotouch-demos-widgetsgallery-l10n-engineering-english"
    "duihomescreen-l10n-engineering-english", # now "meegotouchhome-l10n-engineering-english"
    "keychain-ui-l10n-engineering-english", # does not exist anymore
    "duihomescreen-l10n-engineering-english", # now "meegotouchhome-l10n-engineering-english"
    "mfe-account-ui-plugins-l10n-engineering-english",
    );

# Engineering English packages which should be ignored because they are only demos:
my @engineeringEnglishPackagesIgnore = (
    "meegotouch-demos-widgetsgallery-l10n-engineering-english",
    "duihomescreen-plugins-weatherapplet-l10n-engineering-english"
    );

# Engineering English packages which should be temporarily ignored
# because they are broken, but should be enabled again as soon
# as they are fixed:

my @engineeringEnglishPackagesBroken = (
    # this one is empty. Why?:
    "duicontrolpanel-keyboardlanguageapplet-l10n-engineering-english",
    # this one is empty. Why?:
    "libvideodolbysettings-l10n-engineering-english",
   );

my @engineeringEnglishPackages = ();

my @sourcePackagesToDownloadAndScan = (); # contents autogenerated

# source packages which should be ignored. Possible reasons to ignore:
#   - they have nothing to do with translations
#   - they contain only demos
#   - then cannot be downloaded for whatever reason
#   - other ?
my @sourcePackagesIgnore = (
    "libmeegotouch",
    "meegotouchhome-nokia", # fails to download, 403 error, 2010-10-18
    "meegotouchhome-nokia-ui-tests", # fails to download, 403 error, 2010-10-11
    "nokia-maps", # fails to download, 403 error
    "adobe-flashplayer", # fails to downloadm, 403 error
    "syncshare-test-app", # fails to downloadm, 403 error
    "wrt",        # does this need to be scanned? I guess not.
    "xulrunner",  # does this need to be scanned? I guess not.
    );

# source packages which should be ignored, e.g. because they are obsolete
my @sourcePackagesObsolete = (
    );

my @sourcePackagesScanned = (); # autogenerated from what was really downloaded

#----------------------------------------------------------------------
# subroutines:

sub usage {
    printf STDERR "Usage: photo-gallery [option] ...\n";
    printf STDERR "-v, --verbosity         print some progress messages to standard output.\n";
    printf STDERR "    --version           display version and exit.\n";
    printf STDERR "-h, --help              display this help and exit.\n";
    printf STDERR "    --eedir             directory where the ee .qm files are.\n";
    printf STDERR "    --handoffurl        URL of the “handoff” .ts files.\n";
    printf STDERR "    --handoffdir        directory to save the “handoff” .ts files.\n";
    printf STDERR "    --sourcepackagesdir directory to save the source packages.\n";
    printf STDERR "    --http-user         user name to access the “handoff” .ts files.\n";
    printf STDERR "    --http-passwd       password to access the “handoff” .ts files.\n";
    printf STDERR "                        or use .netrc to store user name and password.\n";
    printf STDERR "    --skip-dowload      skip download.\n";
    printf STDERR "    --output-dir        directory to write the output files to.\n";
    printf STDERR "                        default is ./messageid-check-results/\n";
    exit 1;
}

sub mySystem {
    my ($command) = @_;
    if ($OPT_VERBOSITY >= 1) {
        print "executing: $command\n";
    }
    return system ($command);
}

sub myExit {
    my ($value) =@_;
    chdir $original_wd || die "Can’t cd to $original_wd: $!\n";
    exit ($value);
}

sub myBasename {
    my ($path) =@_;
    $path =~ /\/([^\/]+)$/; # basename (strip directory)
    return $1;
}

sub writeHtml {
    my ($htmlOutBody) =@_;
    my $title = "Results of messageid-check (version $version) run on " . $run_date;

    my $htmlOutHeader = "";
    $htmlOutHeader .= <<"EOF";
<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\">
<html>
<head>
<title>$title</title>
<meta http-equiv=\"content-type\" content=\"text/html; charset=UTF-8\">
<style type=\"text/css\">
body {
  color:#000000;
  background-color:#FFFFFF;
  font-family: sans-serif;
  font-size: 1em;
}
.okcolor {
  color:#00FF00;
}
.errorcolor {
  color:#FF0000;
}
.okbgcolor {
  background-color:#ceeebc;
}
.errorbgcolor {
  background-color:#ffcdcd;
}
.eefile {
  border: solid 1px #88c3ff;
  margin-bottom: 10px;
}
.title {
  background-color: #5a9fe5;
  font-weight: bold;
  padding: 5px;
  padding-left: 15px;
  font-size: 20px;
}
.package {
  background-color: #71b1f2;
  font-weight: bold;
  padding: 5px;
  padding-left: 15px;
  font-size: 20px;
}
table.summary {
  text-align: right;
  font-style: normal;
  font-size: small;
  border: 1;
  margin-top: 0em;
  margin-bottom: 0em;
  margin-left: 0em;
  margin-right: 0em;
  padding-top: 1em;
  padding-bottom: 1em;
  padding-left: 0em;
  padding-right: 0em;
  clear: left;
}
table.summary td {
  color: #000000;
  padding-top: 0em;
  padding-bottom: 0em;
  padding-left: 0.5em;
  padding-right: 0.5em;
}
.maintainer {
  text-align: right;
  font-size: 20px;
  background-color: #88c3ff;
  padding-right: 20px;
  padding-bottom: 1px;
}
.errorlist {
  text-align: left;
  background-color: #ffcdcd;
  padding-left: 20px;
  padding-top: 20px;
  padding-bottom: 1px;
}
.oklist {
  text-align: left;
  background-color: #ceeebc;
  padding-left: 20px;
  padding-top: 20px;
  padding-bottom: 1px;
}
</style>
</head>
<body>
<h1>$title</h1>
EOF

    my $htmlOutFooter = "";
    $htmlOutFooter .= <<"EOF";
</body>
</html>
EOF

    my $htmlOutTotal = $htmlOutHeader . $htmlOutBody . $htmlOutFooter;

    open (HTML, ">$OPT_OUTPUTDIR/messageid-check-result.html") || die "can't open file $OPT_OUTPUTDIR/messageid-check-result.html: $!";
    binmode HTML, ":utf8";
    printf HTML "%s", $htmlOutTotal;
    close (HTML);

}

sub getEngineeringEnglishPackageList {
    my @engineeringEnglishPackagesKnownToApt = ();
    open (PKGLIST, "apt-cache search l10n-engineering-english |")
        || die "Can’t open apt-cache search l10n-engineering-english | : $!";
    binmode PKGLIST, ":utf8";
    while (<PKGLIST>) {
        if ($ARG =~ /(^[^ ]+)/) {
            push (@engineeringEnglishPackagesKnownToApt, $1);
        }
    }
    close (PKGLIST);
    @engineeringEnglishPackages = @engineeringEnglishPackagesKnownToApt;
}

sub download {
    if (mySystem("fakeroot apt-get update")) {
        printf (STDERR "“fakeroot apt-get update” didn’t work.\n");
        exit (1);
    }

    for my $package (@engineeringEnglishPackages) {
        if (grep(/^${package}$/, @engineeringEnglishPackagesObsolete)
            || grep(/^${package}$/, @engineeringEnglishPackagesIgnore)
            || grep(/^${package}$/, @engineeringEnglishPackagesBroken)) {
            if ($OPT_VERBOSITY >=1) {
                printf (STDOUT "do not try to install ignored %s\n", $package);
            }
        }
        else {
            my $command = "fakeroot apt-get -y --force-yes install $package"; 
            if (mySystem($command)) {
                printf (STDERR "“%s” didn’t work.\n", $command);
                exit (1);
            }
            else {
                printf (STDOUT "“%s” OK.\n", $command);
            }
        }
    }

    if (-d "$OPT_SOURCEPACKAGES_DIR") {
        rmtree ("$OPT_SOURCEPACKAGES_DIR", {verbose => 1});
    }
    mkdir ("$OPT_SOURCEPACKAGES_DIR") || die "Can’t mkdir $OPT_SOURCEPACKAGES_DIR: $!\n";
    chdir ("$OPT_SOURCEPACKAGES_DIR") || die "Can’t cd to $OPT_SOURCEPACKAGES_DIR: $!\n";

    for my $sourcePackage (@sourcePackagesToDownloadAndScan) {
        if (grep (/^${sourcePackage}$/, @sourcePackagesIgnore)
            || grep (/^${sourcePackage}$/, @sourcePackagesObsolete)) {
            if ($OPT_VERBOSITY >=1) {
                printf STDOUT "do not try to download ignored source package %s\n",
                        $sourcePackage;
            }
        }
        else {
            my $command = "fakeroot apt-get source $sourcePackage";
            if (mySystem($command)) {
                printf (STDERR "“%s” didn’t work.\n", $command);
                exit (1);
            }
            else {
                printf (STDOUT "“%s” OK.\n", $command);
            }
        }
    }

    chdir $original_wd || die "Can’t cd to $original_wd: $!\n";

    if (-d "$OPT_HANDOFF_DIR") {
        rmtree ("$OPT_HANDOFF_DIR", {verbose => 1});
    }
    mkdir ("$OPT_HANDOFF_DIR") || die "Can’t mkdir $OPT_HANDOFF_DIR: $!\n";
    chdir ("$OPT_HANDOFF_DIR") || die "Can’t cd to $OPT_HANDOFF_DIR: $!\n";

    my $wgetCommand = "wget --no-check-certificate -nd -r -l1 ";
    if ($OPT_HTTP_USER ne "") {
        $wgetCommand .= "--http-user=$OPT_HTTP_USER ";
    }
    if ($OPT_HTTP_PASSWD ne "") {
        $wgetCommand .= "--http-passwd=$OPT_HTTP_PASSWD ";
    }
    $wgetCommand .= "$OPT_HANDOFF_URL";

    if (mySystem($wgetCommand)) {
        printf (STDERR "“wget $OPT_HANDOFF_URL” returned failure code. It does this always when --no-check-certificate is used, so let’s just continue ...\n");
    }

    chdir $original_wd || die "Can’t cd to $original_wd: $!\n";

    @handoffFiles = glob ("$OPT_HANDOFF_DIR/*.ts");
    if ($#handoffFiles < 0) {
        printf (STDERR "download of handoff .ts files failed.\n");
        myExit(1);
    }

    # Convert handoff files mac2unix:
    for my $file (@handoffFiles) {
        printf(STDOUT "mac2unix %s\n", $file);
        my $fileContents = "";
        open (TS, "<:encoding(UTF-8)", "$file") || die "Can’t open file $file: $!";
        while(<TS>) {
            $ARG =~ s/\x{FEFF}//; # remove BOM
            $ARG =~ s/\r/\n/gi;   # replace returns with newlines
            $fileContents .= $ARG;
        }
        close (TS);
        open (TS, ">:encoding(UTF-8)", "$file") || die "Can’t open file $file: $!";
        printf(TS "%s", $fileContents);
        close (TS);
    }
}

sub checkListOfSourcePackagesDependingOnLibmeegotouch {
    my %binaryPackagesDependingOnLibmeegotouch = ();
    my %sourcePackagesDependingOnLibmeegotouch = ();
    open (PKGLIST, "apt-cache rdepends libmeegotouchcore0 |")
        || die "Can’t open apt-cache rdepends libmeegotouchcore0 | : $!";
    binmode PKGLIST, ":utf8";
    while (<PKGLIST>) {
        if ($ARG =~ /libmeegotouchcore0/ || $ARG =~ /Reverse Depends:/) {
            next;
        }
        if ($ARG =~ /^[[:space:]]*([^[:space:]]+)[[:space:]]*$/) {
            if ($OPT_VERBOSITY >=2) {
                printf STDOUT "found binary package %s depending on libmeegotouchcore0\n",
                $1;
            }
            $binaryPackagesDependingOnLibmeegotouch{$1} = "yes";
        }
    }
    close (PKGLIST);
    for my $binaryPackage (sort (keys %binaryPackagesDependingOnLibmeegotouch)) {
        my $sourcePackage = "";
        open (APTSHOW, "apt-cache showsrc $binaryPackage |")
            || die "Can’t open apt-cache show $binaryPackage | : $!";
        while(<APTSHOW>) {
            if ($ARG =~ /Package:\s+(.+)$/) {
                $sourcePackage = $1
            }
            if ($sourcePackage ne "") {
                $sourcePackagesDependingOnLibmeegotouch{$sourcePackage} = "yes";
            }
        }
        close (APTSHOW);
    }
    for my $sourcePackage (sort (keys %sourcePackagesDependingOnLibmeegotouch)) {

        push (@sourcePackagesToDownloadAndScan, $sourcePackage);
    }
}

sub findEngineeringEnglishFiles {
    my %engineeringEnglishPackages = ();
    for my $package (@engineeringEnglishPackages) {
        if (grep(/^${package}$/, @engineeringEnglishPackagesObsolete)
            || grep(/^${package}$/, @engineeringEnglishPackagesIgnore)
            || grep(/^${package}$/, @engineeringEnglishPackagesBroken)) {
            if ($OPT_VERBOSITY >=1) {
                printf (STDOUT "ignoring %s\n", $package);
            }
        }
        else {
            $engineeringEnglishPackages{$package} = "";
        }
    }
    @engineeringEnglishPackages = (sort (keys %engineeringEnglishPackages));
    my @engineeringEnglishFiles = ();
    for my $package (@engineeringEnglishPackages) {
        if ($OPT_VERBOSITY >= 1) {
            printf (STDOUT "using %s\n", $package);
        }
        my @qmFiles = grep (/\.qm$/, qx(dpkg -L $package));
        if ($#qmFiles < 0) {
            printf (STDOUT "ERROR: %s has no .qm files.\n", $package);
            exit (1);
        }
        else {
            for my $file (@qmFiles) {
                chomp ($file);
                if ($OPT_VERBOSITY >= 1) {
                    printf (STDOUT "    %s\n", $file);
                }
                push (@engineeringEnglishFiles, $file);
            }
        }
    }
    return @engineeringEnglishFiles;
}

sub readHandoffXmlTrees {
    for my $handoffFile (glob ("$OPT_HANDOFF_DIR/*.ts")) {
        my $xmlParser = new XML::Parser(Style => 'Tree', ProtocolEncoding => 'UTF-8');
        if ($OPT_VERBOSITY >= 1) {
            printf STDOUT "parsing %s\n", $handoffFile;
        }
        $handoffXmlTrees{$handoffFile} = $xmlParser->parsefile($handoffFile);
    }
}

sub getIdsFromOneContextTreeFromSourcePackages {
    my (@contextTree) = @_;
    #print Dumper(@contextTree);
    if ($contextTree[3] ne "name") {
        printf STDERR "context tree has no name tag. Should not happen, exiting...\n";
        myExit(1);
    }
    my $name = $contextTree[3+1][2];
    if ($name ne "") {
        printf STDOUT "ignoring context tree with non-empty name “%s”\n", $name;
    }
    else {
        for (my $i = 0; $i < $#contextTree; ++$i) {
            if ($contextTree[$i] eq "message") {
                my @messageTree = @{$contextTree[$i+1]};
                my $messageId = $messageTree[0]{"id"};
                my $source = "";
                my $location = "";
                for (my $j = 0; $j < $#messageTree; ++$j) {
                  SWITCH: {
                      if ($messageTree[$j] eq "source") {
                          $source = $messageTree[$j+1][2];
                          last SWITCH;
                      }
                      if ($messageTree[$j] eq "location") {
                          my @locationTree = @{$messageTree[$j+1]};
                          my $filename = $locationTree[0]{"filename"};
                          my $line = $locationTree[0]{"line"};
                          if (!defined $filename) {
                              $filename = "unknown";
                          }
                          if (!defined $line) {
                              $line = "unknown";
                          }
                          if ($location eq "") {
                              $location = "filename=$filename line=$line";
                          }
                          else {
                              $location .= ", filename=$filename line=$line";
                          }
                          last SWITCH;
                      }
                    }
                }
                if (defined $messageId) { # ignore messages which do not have ids
                    $idsFromSourcePackages{$messageId} = $location;
                }
            }
        }
    }
    $totalNumberOfIdsFromSourcePackages = scalar (keys %idsFromSourcePackages);
}

sub getIdsFromHandoffFiles {
    for my $handoffFile (@handoffFiles) {
        my $tree = $handoffXmlTrees{$handoffFile};
        my @tsTree = @{$tree->[1]};
        # print Dumper(@tsTree);
        my $extra_application_info = "";
        my $extra_ui_spec_document = "";
        my $extra_ts_date = "";
        my $extra_ts_macroversion = "";
        my @contextTree = ();
        for (my $i = 0; $i < $#tsTree; ++$i) {
          SWITCH: {
              if ($tsTree[$i] eq "extra-application-info") {
                  $extra_application_info = $tsTree[$i+1][2];
                  last SWITCH;
              }
              if ($tsTree[$i] eq "extra-uispec-document") {
                  $extra_ui_spec_document = $tsTree[$i+1][2];
                  last SWITCH;
              }
              if ($tsTree[$i] eq "extra-ts-date") {
                  $extra_ts_date = $tsTree[$i+1][2];
                  last SWITCH;
              }
              if ($tsTree[$i] eq "extra-ts-macroversion") {
                  $extra_ts_macroversion = $tsTree[$i+1][2];
                  last SWITCH;
              }
              if ($tsTree[$i] eq "context") {
                  @contextTree = @{$tsTree[$i+1]};
                  last SWITCH;
              }
            }
        }
        for (my $i = 0; $i < $#contextTree; ++$i) {
            if ($contextTree[$i] eq "message") {
                my @messageTree = @{$contextTree[$i+1]};
                my $messageId = $messageTree[0]{"id"};
                my $source = "";
                my $extracomment = "";
                for (my $j = 0; $j < $#messageTree; ++$j) {
                  SWITCH: {
                      if ($messageTree[$j] eq "source") {
                          $source = $messageTree[$j+1][2];
                          last SWITCH;
                      }
                      if ($messageTree[$j] eq "extracomment") {
                          $extracomment = $messageTree[$j+1][2];
                          last SWITCH;
                      }
                    }
                }
                my $messageIdSpecDescription = "";
                $messageIdSpecDescription .= "uispec-document=“$extra_ui_spec_document” ";
                $messageIdSpecDescription .= "ts-file=“" . myBasename($handoffFile) . "” ";
                $messageIdSpecDescription .= "ts-date=“$extra_ts_date” ";
                $messageIdSpecDescription .= "source-english=“$source” ";
                $messageIdSpecDescription .= "extracomment=“$extracomment” ";
                    
                if (defined $idsFromHandoffFiles{$messageId}
                    && $idsFromHandoffFiles{$messageId} ne "") {
                    $idsFromHandoffFiles{$messageId} .= " ☺ $messageIdSpecDescription";
                }
                else {
                    $idsFromHandoffFiles{$messageId} = "$messageIdSpecDescription";
                }
                # init this id in the hash used later to check
                # whether this id is used:
                $idsFromSpecUsage{$messageId} = "";
            }
        }
    }
}

sub getIdsFromSourcePackages {
    @sourcePackagesScanned = glob ("$OPT_SOURCEPACKAGES_DIR/*.tar.gz");
    my $sourcePackagesTsFile = "$OPT_SOURCEPACKAGES_DIR/sourcepackages.ts";
    unlink ($sourcePackagesTsFile);
    my $command = "lupdate /usr/bin/lupdate -no-obsolete -locations absolute -no-ui-lines -no-sort -extensions ui,c,c++,cc,cpp,cxx,ch,h,h++,hh,hpp,hxx $OPT_SOURCEPACKAGES_DIR -ts $sourcePackagesTsFile";
    if (mySystem($command)) {
        printf (STDERR "“%s” didn’t work.\n", $command);
        exit (1);
    }
    else {
        printf (STDOUT "“%s” OK.\n", $command);
    }
    my $xmlParser = new XML::Parser(Style => 'Tree',  ProtocolEncoding => 'UTF-8');
    if ($OPT_VERBOSITY >= 1) {
        printf STDOUT "parsing %s\n", $sourcePackagesTsFile;
    }
    my $tree = $xmlParser->parsefile($sourcePackagesTsFile);
    my @tsTree = @{$tree->[1]};
    # print Dumper(@tsTree);
    my @contextTree = ();
    for (my $i = 0; $i < $#tsTree; ++$i) {
      SWITCH: {
          if ($tsTree[$i] eq "context") {
              @contextTree = @{$tsTree[$i+1]};
              # print Dumper(@contextTree);
              getIdsFromOneContextTreeFromSourcePackages(@contextTree);
              last SWITCH;
          }
        }
    }
}

sub checkMessageIdFromEngineeringEnglishAgainstHandoff {
    my ($engineeringEnglishFile, $eeMessageId, $htmlOutRef) = @_;
    my $eeMessageIdIsInSpec = 0;

    if (defined $idsFromHandoffFiles{$eeMessageId}) {
        $eeMessageIdIsInSpec = 1;
        if ($OPT_VERBOSITY >= 2) {
            printf STDOUT "++++++OK: id “%s” spec: %s\n",
            $eeMessageId, $idsFromHandoffFiles{$eeMessageId};
        }
        ${$htmlOutRef} .= "<li>";
        ${$htmlOutRef} .= sprintf "%s<br>\nfound in spec: %s\n",
        $eeMessageId,  $idsFromHandoffFiles{$eeMessageId};
        ${$htmlOutRef} .= "</li>\n";
        my $engineeringEnglishFileBasename =
            myBasename($engineeringEnglishFile);
        if(defined $idsFromSpecUsage{$eeMessageId}
           && $idsFromSpecUsage{$eeMessageId} ne "") {
            $idsFromSpecUsage{$eeMessageId} .=
                ", $engineeringEnglishFileBasename";
        }
        else {
            $idsFromSpecUsage{$eeMessageId}
            = $engineeringEnglishFileBasename;
        }
    }
    else {
        $eeMessageIdIsInSpec = 0;
        if ($OPT_VERBOSITY >= 2) {
            printf STDOUT "***ERROR: id “%s” not found in spec\n", $eeMessageId;
        }
        ${$htmlOutRef} .= "<li>";
        ${$htmlOutRef} .= sprintf "%s", $eeMessageId;
        ${$htmlOutRef} .= "</li>\n";
    }
    return $eeMessageIdIsInSpec;
}

sub compareIdsFromEngineeringEnglishPackagesWithSpec {
    $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
        <center>
        <table class="summary">
        <caption style="caption-side:top">
        Overview of the number of IDs found in Engineering English packages
        and whether they are found or not found in the specifications.
        Click on the file name of the .qm file for details.
        </caption>
        <tr>
        <th>
        Debian package name
        </th>
        <th>
        version
        </th>
        <th>
        Engineering English .qm file
        </th>
        <th>
        IDs in spec
        </th>
        <th>
        Invalid IDs
        </th>
        <th>
        Missing EE
        </th>
        </tr>
EOF

    for my $engineeringEnglishFile (findEngineeringEnglishFiles()) {
        my $errorCount = 0;
        my $okCount = 0;
        my $missingEngineeringEnglishCount = 0;
        my $debianPackage = qx(dpkg -S $engineeringEnglishFile);
        $debianPackage =~ s/:.*$//;
        chomp($debianPackage);
        my $debianPackageMaintainerFull = "Unknown &lt;unknown\@unknown.com%gt;";
        my $debianPackageMaintainerMail = "unknown\@unknown.com";
        my $debianPackageVersion = "";
        my $debianPackageDescription = "";
        open (STATUS, "dpkg -s $debianPackage |") || die "Can’t open dpkg -s $debianPackage | : $!";
        binmode STATUS, ":utf8";
        while (<STATUS>) {
            if ($ARG =~ /Maintainer:\s+(.+)$/) {
                $debianPackageMaintainerFull = $1;
                $debianPackageMaintainerFull =~ /<(.+)>/;
                $debianPackageMaintainerMail = $1;
                $debianPackageMaintainerFull =~ s/</&lt;/;
                $debianPackageMaintainerFull =~ s/>/&gt;/;
            }
            if ($ARG =~ /Version:\s+(.+)$/) {
                $debianPackageVersion = $1;
            }
            if ($ARG =~ /Description:\s+(.+)$/) {
                $debianPackageDescription = $1;
            }
        }
        my $dpkgStatusDetails = qx(dpkg -s $debianPackage);
        if ($OPT_VERBOSITY >= 1) {
            printf STDOUT "------------------------------------------------------------\n";
            printf STDOUT "checking “%s” from package “%s”\n", $engineeringEnglishFile, $debianPackage;
        }

        my $htmlOutOkList = "";
        my $htmlOutErrorList = "";
        my $htmlOutMissingEeList = "";

        my $xmlParser = new XML::Parser(Style => 'Tree');
        open (EEFILE, "lconvert -o - -i $engineeringEnglishFile |") || die "Can’t open file lconvert $engineeringEnglishFile | : $!";
        my $tree = $xmlParser->parse(*EEFILE, ProtocolEncoding => 'UTF-8');
        close(EEFILE);
        if (! $tree->[1][4]) {
            if ($OPT_VERBOSITY >=1) {
                printf STDOUT "%s has no context tree, probably empty, skipping ...\n",
                $engineeringEnglishFile;
            }
            next;
        }
        my @contextTree = @{$tree->[1][4]};
        # print Dumper(@contextTree);
        for (my $i = 0; $i < $#contextTree; ++$i) {
            if ($contextTree[$i] eq "message") {
                my $source = $contextTree[$i+1][4][2];
                $idsFoundInEePackages{$source} = "";
                my $translation = $contextTree[$i+1][8][2];
                if (!$translation || $translation eq "!! " || $translation eq "") {
                    ++$missingEngineeringEnglishCount;
                    ++$totalNumberOfIdsFoundInEePackagesLackingEngineeringEnglish;
                    my $htmlOutMissingEeMessage = "";
                    $htmlOutMissingEeMessage .= "<li>";
                    $htmlOutMissingEeMessage .= sprintf "%s", $source;
                    $htmlOutMissingEeMessage .= "</li>\n";
                    $htmlOutMissingEeList .= $htmlOutMissingEeMessage;
                }
                my $htmlOutMessage = "";
                my $eeMessageIdIsInSpec =
                    checkMessageIdFromEngineeringEnglishAgainstHandoff($engineeringEnglishFile, $source, \$htmlOutMessage);
                if ($eeMessageIdIsInSpec == 1) {
                    ++$okCount;
                    $htmlOutOkList .= $htmlOutMessage;
                }
                else {
                    ++$errorCount;
                    ++$totalNumberOfIdsFoundInEePackagesButNotInSpec;
                    $htmlOutErrorList .= $htmlOutMessage;
                    $csvOut .= "$source,$engineeringEnglishFile,$debianPackage,$debianPackageVersion,$debianPackageMaintainerMail\n";
                }
            }
        }
        $htmlOutIdsFromEePackagesDetail .= <<"EOF";
            <div class="eefile">
            <div class="title">
            <a href="#$engineeringEnglishFile-ref" name="$engineeringEnglishFile">
            $engineeringEnglishFile
            </a>
            IDs in spec:
            <span class="okcolor">
            $okCount
            </span>
EOF

        $class = $errorCount?  "errorcolor" : "okcolor";
        $htmlOutIdsFromEePackagesDetail .= <<"EOF";
            Invalid IDs:
            <span class="$class">
            $errorCount
            </span>
EOF

        $class = $missingEngineeringEnglishCount?  "errorcolor" : "okcolor";
        $htmlOutIdsFromEePackagesDetail .= <<"EOF";
            IDs lacking EE:
            <span class="$class">
            $missingEngineeringEnglishCount
            </span>
            </div>
EOF

        $htmlOutIdsFromEePackagesDetail .= <<"EOF";
            <div class="package">
            $debianPackage $debianPackageVersion
            </div>
            <div class="maintainer">
            <a href="mailto:$debianPackageMaintainerMail">$debianPackageMaintainerFull</a>
            </div>
EOF

        if ($errorCount > 0) {
            $htmlOutIdsFromEePackagesDetail .= <<"EOF";
                <div class="errorlist">
                List of invalid IDs not found in the UI specs:
                <ul>
                $htmlOutErrorList
                </ul>
                </div>
EOF
        }

        if ($missingEngineeringEnglishCount > 0) {
            $htmlOutIdsFromEePackagesDetail .= <<"EOF";
                <div class="errorlist">
                List of IDs where the Engineering English is missing or empty:
                <ul>
                $htmlOutMissingEeList
                </ul>
                </div>
EOF
        }

        if ($okCount > 0) {
            $htmlOutIdsFromEePackagesDetail .= <<"EOF";
                <div class="oklist">
                List of IDs found in the UI specs:
                <ul>
                $htmlOutOkList
                </ul>
                </div>
                </div>
EOF
        }

        # Table of Contents:
        $class = ($errorCount == 0 && $missingEngineeringEnglishCount == 0) ? "okbgcolor" : "errorbgcolor";
        my $engineeringEnglishFileBasename = myBasename($engineeringEnglishFile);
        $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
            <tr>
            <td class="$class">
            $debianPackage
            </td>
            <td class="$class">
            $debianPackageVersion
            </td>
            <td class="$class">
            <a href="#$engineeringEnglishFile" name="$engineeringEnglishFile-ref">
            $engineeringEnglishFileBasename
            </a>:
            </td>
EOF

        $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
            <td class="okbgcolor">
            $okCount
            </td>
EOF

        $class = $errorCount? "errorbgcolor" : "okbgcolor";
        $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
            <td class="$class">
            $errorCount
            </td>
EOF

        $class = $missingEngineeringEnglishCount? "errorbgcolor" : "okbgcolor";
        $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
            <td class="$class">
            $missingEngineeringEnglishCount
            </td>
            </tr>
EOF

    }

    $htmlOutIdsFromEePackagesSummaryTable .= <<"EOF";
        </table>
        </center>
EOF
}

sub checkMessageIdFromSourcePackagesAgainstHandoff {
    my ($sourcePackagesMessageId, $htmlOutRef) = @_;
    my $sourcePackagesMessageIdIsInSpec = 0;

    if (defined $idsFromHandoffFiles{$sourcePackagesMessageId}) {
        $sourcePackagesMessageIdIsInSpec = 1;
        if ($OPT_VERBOSITY >= 1) {
            printf STDOUT "++++++OK: id “%s” spec: %s\n",
            $sourcePackagesMessageId, $idsFromHandoffFiles{$sourcePackagesMessageId};
        }
        ${$htmlOutRef} .= "<li>";
        ${$htmlOutRef} .= sprintf "%s<br>\nfound in source: %s<br>\nfound in spec: %s\n",
        $sourcePackagesMessageId,
        $idsFromSourcePackages{$sourcePackagesMessageId},
        $idsFromHandoffFiles{$sourcePackagesMessageId};
        ${$htmlOutRef} .= "</li>\n";
    }
    else {
        $sourcePackagesMessageIdIsInSpec = 0;
        if ($OPT_VERBOSITY >= 1) {
            printf STDOUT "***ERROR: id “%s” not found in spec\n", $sourcePackagesMessageId;
        }
        ${$htmlOutRef} .= "<li>";
        ${$htmlOutRef} .= sprintf "%s<br>\nfound in source: %s\n",
        $sourcePackagesMessageId,
        $idsFromSourcePackages{$sourcePackagesMessageId};
        ${$htmlOutRef} .= "</li>\n";
    }
    return $sourcePackagesMessageIdIsInSpec;
}

sub compareIdsFromSourcePackagesWithSpec {
    my $errorCount = 0;
    my $okCount = 0;
    my $htmlOutOkList = "";
    my $htmlOutErrorList = "";
    for my $sourcePackagesMessageId (sort keys %idsFromSourcePackages) {
        if ($OPT_VERBOSITY >= 1) {
            printf STDOUT "checking: “%s” from source “%s”\n",
            $sourcePackagesMessageId,
            $idsFromSourcePackages{$sourcePackagesMessageId};
        }
        my $htmlOutMessage = "";
        my $sourcePackagesMessageIdIsInSpec =
            checkMessageIdFromSourcePackagesAgainstHandoff($sourcePackagesMessageId, \$htmlOutMessage);
        if ($sourcePackagesMessageIdIsInSpec == 1) {
            ++$okCount;
            $htmlOutOkList .= $htmlOutMessage;
        }
        else {
            ++$errorCount;
            ++$totalNumberOfIdsFoundInSourcePackagesButNotInSpec;
            $htmlOutErrorList .= $htmlOutMessage;
        }
    }

    $htmlOutIdsFromSourcePackagesDetail .= <<"EOF";
        <div class="eefile">
        <div class="title">
        Total number of IDs found in source packages:
        ${totalNumberOfIdsFromSourcePackages}.
        IDs in spec:
        <span class="okcolor">
        $okCount
        </span>
        Invalid IDs:
        <span class="errorcolor">
        $errorCount
        </span>
        <a href="#summary" name="idsfromsourcepackagesdetails">
        Back to Summary
        </a>
        </div>
        </div>
EOF
    my $sourcePackagesScannedList = "";
    for my $package (@sourcePackagesScanned) {
        $package =~ s/$OPT_SOURCEPACKAGES_DIR\///g;
        $package =~ s/\.tar\.gz//g;
        $sourcePackagesScannedList .= "<li>$package</li>\n";
    }
    my $numberOfSourcePackagesScanned = $#sourcePackagesScanned + 1;
    $htmlOutIdsFromSourcePackagesDetail .= <<"EOF";
        List of source packages scanned (number of packages scanned: $numberOfSourcePackagesScanned):
        <ul>
            $sourcePackagesScannedList
        </ul>
EOF
 
    if ($errorCount > 0) {
        $htmlOutIdsFromSourcePackagesDetail .= <<"EOF";
                <div class="errorlist">
                List of invalid IDs found in the source packages but <b>not</b> in the UI specs:
                <ul>
                $htmlOutErrorList
                </ul>
                </div>
EOF
    }

    if ($okCount > 0) {
        $htmlOutIdsFromSourcePackagesDetail .= <<"EOF";
                <div class="oklist">
                List of IDs found in the source packages which <b>are</b> in the UI specs:
                <ul>
                $htmlOutOkList
                </ul>
                </div>
EOF
    }
}

#----------------------------------------------------------------------
# "main":

# Process command line options
my %opt;
unless (GetOptions(\%opt,
		   'verbosity|v=i',      \$OPT_VERBOSITY,
		   'version',            \$OPT_VERSION,
		   'help|h',             \$OPT_HELP,
		   'eedir=s',            \$OPT_EEDIR,
                   'handoffurl=s',       \$OPT_HANDOFF_URL,
                   'handoffdir=s',       \$OPT_HANDOFF_DIR,
                   'sourcepackagesdir=s',\$OPT_SOURCEPACKAGES_DIR,
                   'http-user=s',        \$OPT_HTTP_USER,
                   'http-passwd=s',      \$OPT_HTTP_PASSWD,
                   'skip-download',      \$OPT_SKIP_DOWNLOAD,
                   'output-dir=s',       \$OPT_OUTPUTDIR,
        )) {
    &usage ();
    exit 1;
}

if (!defined $OPT_VERBOSITY) {
    $OPT_VERBOSITY = 1;
}
if (!defined $OPT_VERSION) {
    $OPT_VERSION = 0;
}
if (!defined $OPT_HELP) {
    $OPT_HELP = 0;
}
if (!defined $OPT_EEDIR) {
    $OPT_EEDIR="/usr/share/l10n/meegotouch";
}
if (!defined $OPT_HANDOFF_URL) {
    $OPT_HANDOFF_URL="https://projects.maemo.org/svn/l10n/projects/harmattan/vendor/handoff/";
}
if (!defined $OPT_HANDOFF_DIR) {
    $OPT_HANDOFF_DIR="/tmp/messageid-check/handoff-${run_date}";
}
if (!defined $OPT_SOURCEPACKAGES_DIR) {
    $OPT_SOURCEPACKAGES_DIR="/tmp/messageid-check/sourcepackages-${run_date}";
}
if (!defined $OPT_HTTP_USER) {
    $OPT_HTTP_USER="";
}
if (!defined $OPT_HTTP_PASSWD) {
    $OPT_HTTP_PASSWD="";
}
if (!defined $OPT_SKIP_DOWNLOAD) {
    $OPT_SKIP_DOWNLOAD = 0;
}
if (!defined $OPT_OUTPUTDIR) {
    $OPT_OUTPUTDIR = "$original_wd/messageid-check-results/";
}

if ($OPT_VERSION) {
    print "messageid-check $version\n";
    exit 0;
}

if ($OPT_HELP) {
    &usage ();
    exit 0;
}

if ($OPT_VERBOSITY >=1) {
    select (STDOUT);
    $OUTPUT_AUTOFLUSH = 1;
    select (STDERR);
    $OUTPUT_AUTOFLUSH = 1;
}

if (! -d $OPT_OUTPUTDIR) {
    mySystem("mkdir -p $OPT_OUTPUTDIR");
    if (! -d $OPT_OUTPUTDIR) {
        printf STDERR "Cannot create output directory %s\n", $OPT_OUTPUTDIR;
        myExit(1);
    }
}

getEngineeringEnglishPackageList();

if (!$OPT_SKIP_DOWNLOAD) {
    checkListOfSourcePackagesDependingOnLibmeegotouch();
    download();
}
else {
    @handoffFiles = glob ("$OPT_HANDOFF_DIR/*.ts");
}

checkListOfSourcePackagesDependingOnLibmeegotouch();

readHandoffXmlTrees();

getIdsFromHandoffFiles();

getIdsFromSourcePackages();

compareIdsFromEngineeringEnglishPackagesWithSpec();

compareIdsFromSourcePackagesWithSpec();

######################################################################
# generate output to show which ids from specs are used and which are not

# mark some exceptions for IDs where neither Engineering English
# nor sources are expected to exist and which are known to be used anyway:
for my $idFromSpec (sort (keys %idsFromSpecUsage)) {
    if($idsFromSpecUsage{$idFromSpec} eq "") {
        if ($idFromSpec =~ /qtn_clk_city/
            || $idFromSpec =~ /qtn_clk_country/
            || $idFromSpec =~ /qtn_clk_region/) {
            $idsFromSpecUsage{$idFromSpec} = "meegotouch-cities-template";
        }
    }
}

# mark the ids found by scanning the source debian packages
# with lupdate as used:
for my $idFromSpec (sort (keys %idsFromSpecUsage)) {
    if(defined $idsFromSourcePackages{$idFromSpec}
       && $idsFromSourcePackages{$idFromSpec} ne "") {
        if ($idsFromSpecUsage{$idFromSpec} ne "") {
            $idsFromSpecUsage{$idFromSpec} .=
                ", $idsFromSourcePackages{$idFromSpec}";
        }
        else {
            $idsFromSpecUsage{$idFromSpec} =
                $idsFromSourcePackages{$idFromSpec};
        }
    }
}

my $htmlOutIdsFromSpecUsed = "";
my $htmlOutIdsFromSpecNeverUsed = "";
for my $idFromSpec (sort (keys %idsFromSpecUsage)) {
    if ($idsFromSpecUsage{$idFromSpec} eq "") {
        ++$totalNumberOfIdsFoundInSpecButNeverUsed;
        $htmlOutIdsFromSpecNeverUsed .= <<"EOF"
            <li>$idFromSpec</li>
EOF
    }
    else {
        ++$totalNumberOfIdsFoundInSpecUsed;
        $htmlOutIdsFromSpecUsed .= <<"EOF"
            <li>$idFromSpec (used in $idsFromSpecUsage{$idFromSpec})</li>
EOF
    }
}
######################################################################
$totalNumberOfIdsFoundInSpec =
    $totalNumberOfIdsFoundInSpecUsed
    + $totalNumberOfIdsFoundInSpecButNeverUsed;

if (scalar(keys %idsFromSpecUsage) != $totalNumberOfIdsFoundInSpec) {
    printf STDERR "There is something wrong with calculating the number of ids in the specs. Exit.\n";
    myExit(1);
}
if (scalar(keys %idsFromHandoffFiles) != $totalNumberOfIdsFoundInSpec) {
    printf STDERR "There is something wrong with calculating the number of ids in the specs. Exit.\n";
    myExit(1);
}


my $htmlOutIdsFromSpecUsedDetail = "";
$htmlOutIdsFromSpecUsedDetail .= <<"EOF";
    <div class="eefile">
    <div class="title">
    Total number of IDs in specs:
    <span class="okcolor">
    $totalNumberOfIdsFoundInSpec
    </span>
    IDs used:
    <span class="okcolor">
    $totalNumberOfIdsFoundInSpecUsed
    </span>
    IDs never used:
    <span class="errorcolor">
    $totalNumberOfIdsFoundInSpecButNeverUsed
    </span>
    <a href="#summary" name="idsfromspecused">
    Back to Summary
    </a>
    </div>
    </div>
EOF

if ($totalNumberOfIdsFoundInSpecButNeverUsed > 0) {
    $htmlOutIdsFromSpecUsedDetail .= <<"EOF";
        <div class="errorlist">
        List of IDs from the specifications which are never used
        in the code, they can be found neither in the Engineering English
        packages scanned nor in the source packages scanned:
        <ul>
        $htmlOutIdsFromSpecNeverUsed
        </ul>
        </div>
EOF
}
if ($totalNumberOfIdsFoundInSpecUsed > 0) {
    $htmlOutIdsFromSpecUsedDetail .= <<"EOF";
        <div class="oklist">
        List of IDs from the specifications which are used somewhere in the code,
        i.e. they were found in an Engineering English package scanned or in a
        source package scanned or they are known to be used and handled as exceptions:
        <ul>
        $htmlOutIdsFromSpecUsed
        </ul>
        </div>
EOF
}
######################################################################

my $htmlOutTocHeader = "";
$htmlOutTocHeader .= <<"EOF";
    <hr>
    <h2 id="summary">Summary of results:</h2>
EOF

my @idsFoundInEePackagesArray = (keys %idsFoundInEePackages);
$totalNumberOfIdsFoundInEePackages = $#idsFoundInEePackagesArray;

$htmlOutTocHeader .= <<"EOF";
    <h3>Ids found in Engineering English packages:</h3>
    <p>
    <b>
    Total number of Ids found in the Engineering English packages:
    $totalNumberOfIdsFoundInEePackages
    </b>
    (may contain duplicates)
    </p>
EOF

$class = $totalNumberOfIdsFoundInEePackagesButNotInSpec? "errorcolor" : "okcolor";
$htmlOutTocHeader .= <<"EOF";
    <p>
    <b>
    Total number of Ids found in the Engineering English packages but not in the specifications:
    <span class="$class">
    $totalNumberOfIdsFoundInEePackagesButNotInSpec
    </span>
    </b>
    (may contain duplicates)
    </p>
EOF

$class = $totalNumberOfIdsFoundInEePackagesLackingEngineeringEnglish? "errorcolor" : "okcolor";
$htmlOutTocHeader .= <<"EOF";
    <p>
    <b>
    Total number of Ids found in the Engineering English packages where the Engineering English is missing or empty:
    <span class="$class">
    $totalNumberOfIdsFoundInEePackagesLackingEngineeringEnglish
    </span>
    </b>
    (may contain duplicates)
    </p>
EOF

$htmlOutTocHeader .= <<"EOF";
    <p>
    Note that the numbers of Ids listed above as
    “found in the Engineering English packages” may
    count identical ids several times. If the same id
    appears in different Engineering English files it is
    counted once for each Engineering English file it appears in.
    </p>
EOF

$htmlOutTocHeader .= <<"EOF";
    <h3>
    Ids found by scanning source packages:
    <a href="#idsfromsourcepackagesdetails">View source packages scan details</a>
    </h3>
    <p>
    <b>
    Total number of Ids found by scanning source packages with lupdate:
    $totalNumberOfIdsFromSourcePackages
    </b>
    (unique)
    </p>
EOF

$class = $totalNumberOfIdsFoundInSourcePackagesButNotInSpec? "errorcolor" : "okcolor";
$htmlOutTocHeader .= <<"EOF";
    <p>
    <b>
    Total number of Ids found by scanning the source packages which are not in the specifications:
    <span class="$class">
    $totalNumberOfIdsFoundInSourcePackagesButNotInSpec
    </span>
    </b>
    (unique)
    </p>
EOF

$htmlOutTocHeader .= <<"EOF";
    <h3>
    Ids found by scanning handoff files (i.e. the specifications):
    <a href="#idsfromspecused">View handoff files scan details</a>
    </h3>
    <p>
    <b>
    Total number of Ids found in the specifications:
    $totalNumberOfIdsFoundInSpec
    </b>
    (unique)
    </p>
EOF

$class = $totalNumberOfIdsFoundInSpecButNeverUsed? "errorcolor" : "okcolor";
$htmlOutTocHeader .= <<"EOF";
    <p>
    <b>
    Total number of Ids found in the specifications which are never used in the code:
    <span class="$class">
    $totalNumberOfIdsFoundInSpecButNeverUsed
    </span>
    </b>
    (unique)
    </p>  
EOF

$class = $totalNumberOfIdsFoundInSpecUsed? "okcolor" : "errorcolor";
$htmlOutTocHeader .= <<"EOF";
    <p>
    <b>
    Total number of unique Ids found in the specifications which are used somewhere in the code:
    <span class="$class">
    $totalNumberOfIdsFoundInSpecUsed
    </span>
    </b>
    (unique)
    </p>  
EOF

$htmlOutTocHeader .= <<"EOF";
    <p>
    <a href="messageid-check-result.csv">
    CSV file with a list of invalid message IDs from Engineering English Packages.
    </a>
    (does not include the invalid message IDs found from scanning the sources directly).
    </p>
EOF

my $htmlOutResultsDetailHeader = "";
$htmlOutResultsDetailHeader .= <<"EOF";
    <hr>
    <h2>Results in detail:</h2>
EOF

my $htmlOutIntroduction ="";
$htmlOutIntroduction .= <<"EOF";
    <hr>
    <h2>Introduction</h2>
    <p>
    This page lists IDs which are used in
    the code but which are missing in the specifications. For all IDs used
    in the code, but not defined in the UI specifications, please do the
    following:
    </p>
    <ol>
    <li>Check your code if those IDs are really in use</li>
    <li>Check if there is a typo in the ID</li>
    </ol>
    <p>
    If there is no typo and the ID is
    really used, open a bug against the relevant UI Spec
    requesting a new ID:
    </p>
    <ul>
    <li>component: Specifications</li>
    <li>CC: <a href=\"mailto:heidi.hendrell\@nokia.com\">Heidi Hendrell</a></li>
    </ul>
    <p>
    The message ids which are in the specifications were collected
    from the “handoff” .ts files which were downloaded here:<br>
    <a href="$OPT_HANDOFF_URL">$OPT_HANDOFF_URL</a>.
    </p>
EOF

my $etc_apt_sources_list_contents = "";
open (SOURCES_LIST, "<:encoding(UTF-8)", "/etc/apt/sources.list") || die "Can’t open file /etc/apt/sources.list: $!";
while (<SOURCES_LIST>) {
    next if /^\s*#/; # skip comments
    if ($ARG =~ /[^[:space:]]+/) {
        $etc_apt_sources_list_contents .= "$ARG<br>";
    }
}
close (SOURCES_LIST);

$htmlOutIntroduction .= <<"EOF";
    <p>
    While this script was run, /etc/apt/sources.list contained:<br>
    $etc_apt_sources_list_contents
    i.e. the Engineering English packages and source packages
    listed below were downloaded from these repositories.
    </p>
EOF

my $htmlOutBody =
    $htmlOutIntroduction
    . $htmlOutTocHeader
    . $htmlOutIdsFromEePackagesSummaryTable
    . $htmlOutIdsFromSourcePackagesSummaryTable
    . $htmlOutResultsDetailHeader
    . $htmlOutIdsFromEePackagesDetail
    . $htmlOutIdsFromSourcePackagesDetail
    . $htmlOutIdsFromSpecUsedDetail;
writeHtml($htmlOutBody);

open (CSV, ">$OPT_OUTPUTDIR/messageid-check-result.csv") || die "can't open file $OPT_OUTPUTDIR/messageid-check-result.csv: $!";
binmode CSV, ":utf8";
printf CSV "%s", $csvOut;
close (CSV);

myExit (0);