From 1c6b1f3b982a6c8ed5e85d51d1c8b090be7ec53e Mon Sep 17 00:00:00 2001 From: Cynthia Date: Fri, 3 Jun 2022 20:16:08 +0200 Subject: [PATCH 1/3] fix(report/attachment): use gzip instead of zip --- reports/opendmarc-reports.in | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/reports/opendmarc-reports.in b/reports/opendmarc-reports.in index 69a2194b..eebb5dfb 100755 --- a/reports/opendmarc-reports.in +++ b/reports/opendmarc-reports.in @@ -20,7 +20,7 @@ use File::Temp; use Net::Domain qw(hostfqdn hostdomain); use Getopt::Long; use IO::Handle; -use IO::Compress::Zip qw(zip); +use IO::Compress::Gzip qw(gzip); use POSIX; use MIME::Base64; use Net::SMTP; @@ -50,7 +50,7 @@ my $boundary; my $tmpout; my $repfile; -my $zipfile; +my $gzipfile; my $zipin; @@ -517,7 +517,7 @@ foreach (@$domainset) # construct the temporary file $repfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".xml"; - $zipfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".zip"; + $gzipfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".xml.gz"; if (!open($tmpout, ">", $repfile)) { print STDERR "$progname: can't create report file for domain $domain\n"; @@ -871,9 +871,9 @@ foreach (@$domainset) } # zip the report - if (!zip [ $repfile ] => $zipfile) + if (!gzip [ $repfile ] => $gzipfile) { - print STDERR "$progname: can't zip report for domain $domain: $!\n"; + print STDERR "$progname: can't gzip report for domain $domain: $!\n"; next; } @@ -947,9 +947,9 @@ foreach (@$domainset) my $datestr; my $report_id; - if (!open($zipin, $zipfile)) + if (!open($zipin, $gzipfile)) { - print STDERR "$progname: can't read zipped report for $domain: $!\n"; + print STDERR "$progname: can't read gzipped report for $domain: $!\n"; next; } @@ -978,8 +978,8 @@ foreach (@$domainset) $mailout .= "generated at " . localtime() . "\n"; $mailout .= "\n"; $mailout .= "--$boundary\n"; - $mailout .= "Content-Type: application/zip\n"; - $mailout .= "Content-Disposition: attachment; filename=\"$zipfile\"\n"; + $mailout .= "Content-Type: application/gzip\n"; + $mailout .= "Content-Disposition: attachment; filename=\"$gzipfile\"\n"; $mailout .= "Content-Transfer-Encoding: base64\n"; $mailout .= "\n"; @@ -1044,7 +1044,7 @@ foreach (@$domainset) } } - unlink($zipfile); + unlink($gzipfile); if (!$keepfiles) { unlink($repfile); From 049221babe3840c0ab8e24781a17157b529c876f Mon Sep 17 00:00:00 2001 From: Cynthia Date: Fri, 3 Jun 2022 22:31:32 +0200 Subject: [PATCH 2/3] fix!: add missing fields to aggregate reports BREAKING CHANGE: MySQL schema has been updated as additional data needs to be stored. fixes #52 --- db/schema.mysql | 2 ++ opendmarc/opendmarc.c | 7 +++++ reports/opendmarc-import.in | 22 ++++++++++++---- reports/opendmarc-reports.in | 51 ++++++++++++++++++++++++++---------- 4 files changed, 63 insertions(+), 19 deletions(-) diff --git a/db/schema.mysql b/db/schema.mysql index 059c3de2..fadcb5e8 100644 --- a/db/schema.mysql +++ b/db/schema.mysql @@ -64,6 +64,7 @@ CREATE TABLE IF NOT EXISTS requests ( policy TINYINT NOT NULL DEFAULT '0', spolicy TINYINT NOT NULL DEFAULT '0', pct TINYINT NOT NULL DEFAULT '0', + fo TINYINT NOT NULL DEFAULT '0', locked TINYINT NOT NULL DEFAULT '0', firstseen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, lastsent TIMESTAMP NOT NULL DEFAULT '1970-01-01 00:00:00', @@ -106,6 +107,7 @@ CREATE TABLE IF NOT EXISTS messages ( from_domain INT UNSIGNED NOT NULL, policy_domain INT UNSIGNED NOT NULL, spf TINYINT NOT NULL, + spf_scope TINYINT NOT NULL, align_dkim TINYINT UNSIGNED NOT NULL, align_spf TINYINT UNSIGNED NOT NULL, sigcount TINYINT UNSIGNED NOT NULL, diff --git a/opendmarc/opendmarc.c b/opendmarc/opendmarc.c index aee0d484..b667bc92 100644 --- a/opendmarc/opendmarc.c +++ b/opendmarc/opendmarc.c @@ -2314,6 +2314,7 @@ mlfi_eom(SMFICTX *ctx) int adkim; int aspf; int pct; + int fo; int p; int sp; int align_dkim; @@ -2856,6 +2857,9 @@ mlfi_eom(SMFICTX *ctx) dmarcf_dstring_printf(dfc->mctx_histbuf, "spf %d\n", dfc->mctx_spfresult); + dmarcf_dstring_printf(dfc->mctx_histbuf, + "spf_scope %d\n", + spfmode); wspf = TRUE; } else if (ar.ares_result[c].result_method == ARES_METHOD_DKIM) @@ -3245,6 +3249,9 @@ mlfi_eom(SMFICTX *ctx) opendmarc_policy_fetch_pct(cc->cctx_dmarc, &pct); dmarcf_dstring_printf(dfc->mctx_histbuf, "pct %d\n", pct); + opendmarc_policy_fetch_fo(cc->cctx_dmarc, &fo); + dmarcf_dstring_printf(dfc->mctx_histbuf, "fo %d\n", fo); + opendmarc_policy_fetch_adkim(cc->cctx_dmarc, &adkim); dmarcf_dstring_printf(dfc->mctx_histbuf, "adkim %d\n", adkim); diff --git a/reports/opendmarc-import.in b/reports/opendmarc-import.in index 3a2f404e..2ba5cec8 100755 --- a/reports/opendmarc-import.in +++ b/reports/opendmarc-import.in @@ -77,6 +77,7 @@ my $ipaddr; my $jobid; my $p; my $pct; +my $fo; my $pdomain; my $policy; my $received; @@ -85,6 +86,7 @@ my $repuri; my $sigcount = 0; my $sp; my $spf; +my $spf_scope; my @rua; ### @@ -248,16 +250,16 @@ sub update_db $dbi_s = $dbi_h->prepare(q{ INSERT INTO messages( date, jobid, reporter, policy, disp, ip, env_domain, from_domain, - policy_domain, spf, align_spf, align_dkim, sigcount, arc, arc_policy + policy_domain, spf, spf_scope, align_spf, align_dkim, sigcount, arc, arc_policy ) VALUES( - FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) }); if (!$dbi_s->execute( $received, $jobid, $rep_id, $policy, $action, $ipaddr_id, $envfrom_id, $from_id, - $pdomain_id, $spf, $align_spf, $align_dkim, $sigcount, $arc, $arc_policy + $pdomain_id, $spf, $spf_scope, $align_spf, $align_dkim, $sigcount, $arc, $arc_policy )) { print STDERR "$progname: failed to insert message: " . $dbi_h->errstr . "\n"; @@ -400,9 +402,9 @@ sub update_db $dbi_s->finish; } - $dbi_s = $dbi_h->prepare("UPDATE requests SET adkim = ?, aspf = ?, policy = ?, spolicy = ?, pct = ? WHERE id = ?"); + $dbi_s = $dbi_h->prepare("UPDATE requests SET adkim = ?, aspf = ?, policy = ?, spolicy = ?, pct = ?, fo = ? WHERE id = ?"); - if (!$dbi_s->execute($adkim, $aspf, $p, $sp, $pct, $request_id)) + if (!$dbi_s->execute($adkim, $aspf, $p, $sp, $pct, $fo, $request_id)) { print STDERR "$progname: failed to update policy data for $fdomain: " . $dbi_h->errstr . "\n"; $dbi_s->finish; @@ -673,6 +675,7 @@ while (<$inputfh>) undef $jobid; undef $p; undef $pct; + undef $fo; undef $pdomain; undef $policy; undef $received; @@ -681,6 +684,7 @@ while (<$inputfh>) $sigcount = 0; undef $sp; undef $spf; + undef $spf_scope; } $jobid = $value; @@ -702,6 +706,10 @@ while (<$inputfh>) $pct = $value; } + case "fo" { + $fo = $value; + } + case "pdomain" { $pdomain = $value; } @@ -733,6 +741,10 @@ while (<$inputfh>) $spf = $value; } + case "spf_scope" { + $spf_scope = $value; + } + else { print STDERR "$progname: unknown key '$key' at line $lineno\n"; } diff --git a/reports/opendmarc-reports.in b/reports/opendmarc-reports.in index eebb5dfb..189c8326 100755 --- a/reports/opendmarc-reports.in +++ b/reports/opendmarc-reports.in @@ -70,6 +70,7 @@ my $spolicy; my $policystr; my $spolicystr; my $pct; +my $fo; my $repuri; my @repuris; @@ -84,9 +85,11 @@ my $align_dkimstr; my $align_spf; my $align_spfstr; my $spfresult; +my $spfscope; my $dkimresult; my $disp; my $spfresultstr; +my $spfscopestr; my $dkimresultstr; my $dispstr; my $ipaddr; @@ -447,7 +450,7 @@ foreach (@$domainset) next; } - $dbi_s = $dbi_h->prepare("SELECT repuri, adkim, aspf, policy, spolicy, pct, UNIX_TIMESTAMP(lastsent) FROM requests WHERE domain = ?"); + $dbi_s = $dbi_h->prepare("SELECT repuri, adkim, aspf, policy, spolicy, pct, fo, UNIX_TIMESTAMP(lastsent) FROM requests WHERE domain = ?"); if (!$dbi_s->execute($domainid)) { print STDERR "$progname: can't get reporting URI for domain $domain: " . $dbi_h->errstr . "\n"; @@ -486,7 +489,11 @@ foreach (@$domainset) } if (defined($dbi_a->[6])) { - $lastsent = $dbi_a->[6]; + $fo = $dbi_a->[6]; + } + if (defined($dbi_a->[7])) + { + $lastsent = $dbi_a->[7]; } } @@ -559,14 +566,16 @@ foreach (@$domainset) print $tmpout "\n"; print $tmpout "\n"; + print $tmpout " 1\n"; + print $tmpout " \n"; print $tmpout " $repdom\n"; - print $tmpout " $repemail\n"; - print $tmpout " $domain:$now\n"; - print $tmpout " \n"; - print $tmpout " $repstart\n"; - print $tmpout " $repend\n"; - print $tmpout " \n"; + print $tmpout " $repemail\n"; + print $tmpout " $domain:$now\n"; + print $tmpout " \n"; + print $tmpout " $repstart\n"; + print $tmpout " $repend\n"; + print $tmpout " \n"; print $tmpout " \n"; print $tmpout " \n"; @@ -576,6 +585,7 @@ foreach (@$domainset) print $tmpout "

$policystr

\n"; print $tmpout " $spolicystr\n"; print $tmpout " $pct\n"; + print $tmpout " $fo\n"; print $tmpout " \n"; if ($daybound) @@ -597,8 +607,8 @@ foreach (@$domainset) { $dbi_s = $dbi_h->prepare(q{ SELECT messages.id, ipaddr.addr, messages.disp, d1.name, d2.name, - messages.spf, messages.align_spf, messages.align_dkim, - messages.arc, messages.arc_policy + messages.spf, messages.spfscope, messages.align_spf, + messages.align_dkim, messages.arc, messages.arc_policy FROM messages JOIN ipaddr ON messages.ip = ipaddr.id JOIN domains d1 ON messages.from_domain = d1.id @@ -649,19 +659,23 @@ foreach (@$domainset) } if (defined($dbi_a->[6])) { - $align_spf = $dbi_a->[6]; + $spfscope = $dbi_a->[6]; } if (defined($dbi_a->[7])) { - $align_dkim = $dbi_a->[7]; + $align_spf = $dbi_a->[7]; } if (defined($dbi_a->[8])) { - $arc = $dbi_a->[8]; + $align_dkim = $dbi_a->[8]; } if (defined($dbi_a->[9])) { - $arcpolicy = $dbi_a->[9]; + $arc = $dbi_a->[9]; + } + if (defined($dbi_a->[10])) + { + $arcpolicy = $dbi_a->[10]; } if (!defined($msgid)) @@ -696,6 +710,13 @@ foreach (@$domainset) else { $spfresultstr = "unknown"; } } + switch ($spfscope) + { + case 1 { $spfscopestr = "mfrom"; } + case 2 { $spfscopestr = "helo"; } + else { $spfscopestr = "unknown"; } + } + switch ($align_dkim) { case 4 { $align_dkimstr = "pass"; } @@ -772,10 +793,12 @@ foreach (@$domainset) print $tmpout " \n"; print $tmpout " \n"; print $tmpout " \n"; + print $tmpout " $envdomain\n"; print $tmpout " $fromdomain\n"; print $tmpout " \n"; print $tmpout " \n"; print $tmpout " \n"; + print $tmpout " $spfscopestr\n"; print $tmpout " $envdomain\n"; print $tmpout " $spfresultstr\n"; print $tmpout " \n"; From 08bf3e1e08c4a8225190a96141f1d6a6494ad3e5 Mon Sep 17 00:00:00 2001 From: Cynthia Date: Fri, 3 Jun 2022 22:56:34 +0200 Subject: [PATCH 3/3] style: consistent use of tabs/space in sql string --- reports/opendmarc-reports.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reports/opendmarc-reports.in b/reports/opendmarc-reports.in index 189c8326..a06569d3 100755 --- a/reports/opendmarc-reports.in +++ b/reports/opendmarc-reports.in @@ -608,7 +608,7 @@ foreach (@$domainset) $dbi_s = $dbi_h->prepare(q{ SELECT messages.id, ipaddr.addr, messages.disp, d1.name, d2.name, messages.spf, messages.spfscope, messages.align_spf, - messages.align_dkim, messages.arc, messages.arc_policy + messages.align_dkim, messages.arc, messages.arc_policy FROM messages JOIN ipaddr ON messages.ip = ipaddr.id JOIN domains d1 ON messages.from_domain = d1.id