about: status update about gpg tools

Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 authors = ["Kai Michaelis <kai@sequoia-pgp.org>"]
 build = "build.rs"
 default-run = "hagrid"
+edition = "2018"
 
 [workspace]
 members = [
@@ -32,6 +33,8 @@ num_cpus = "1.0"
 ring = "0.13"
 base64 = "0.10"
 uuid = "0.7"
+rocket_prometheus = "0.2"
+lazy_static = "1.3.0"
 
 [dependencies.lettre]
 version = "0.9"

Rocket.toml.dist

@@ -16,6 +16,7 @@ token_dir = "state/tokens"
 tmp_dir = "state/tmp"
 mail_rate_limit = 60
 maintenance_file = "state/maintenance"
+enable_prometheus = false
 
 [staging]
 base-URI = "https://keys.openpgp.org"
@@ -31,6 +32,7 @@ token_dir = "tokens"
 tmp_dir = "tmp"
 mail_rate_limit = 60
 maintenance_file = "maintenance"
+enable_prometheus = false
 
 [production]
 base-URI = "https://keys.openpgp.org"
@@ -47,3 +49,4 @@ token_dir = "tokens"
 tmp_dir = "tmp"
 mail_rate_limit = 3600
 maintenance_file = "maintenance"
+enable_prometheus = false

dist/assets/js/upload.js

@@ -0,0 +1,14 @@
+var body = document.getElementsByTagName("body")[0];
+
+  body.addEventListener('dragover', function(e) {
+    e.preventDefault();
+    e.stopPropagation();
+  }, false);
+
+  body.addEventListener('drop', function(e) {
+    e.preventDefault();
+    e.stopPropagation();
+    if (e.dataTransfer) {
+      document.getElementById('keytext').files = e.dataTransfer.files;
+    }
+}, false);

dist/assets/site.css

@@ -43,11 +43,23 @@ a.brand {
   color: #050505;
 }
 
-h3:target {
+.usage h2 div {
+  display: inline-block;
+  width: 2.5em;
+  text-align: center;
+}
+
+.usage h2 div img {
+  display: inline-block;
+  height: 2em;
+  vertical-align: -30%;
+}
+
+h4:target, h3:target {
   background-color: #ffa;
 }
 
-h3 a, h3 a:visited {
+h4 a, h4 a:visited, h3 a, h3 a:visited {
   color: #050505;
 }
 

dist/templates/about/usage.html.hbs

@@ -10,35 +10,67 @@
         write to us and we'll try to add it.
     </p>
 
-    <h2 style="margin-left: 3%;"><img src="/assets/img/enigmail.svg" style="display: inline; height: 2em; vertical-align: -30%; padding-right: 10px;" /> Enigmail</h2>
+    <h2>
+        <div><img src="/assets/img/enigmail.svg"></div>
+        Enigmail
+    </h2>
     <p>
-        Enigmail will support <span class="brand">keys.openpgp.org</span> out
-        of the box in an upcoming version. Stay tuned!
+        <a href="https://enigmail.net" target="_blank">Enigmail</a> for Thunderbird
+        uses <span class="brand">keys.openpgp.org</span> by default since
+        version 2.0.12.
+    </p>
+    <p>Full support is available since Enigmail 2.1
+        (for <a href="https://www.thunderbird.net/en-US/thunderbird/68.0beta/releasenotes/" target="_blank">Thunderbird 68</a> or newer):
         <ul>
             <li>Keys will be kept up to date automatically.</li>
-            <li>During creation, you can optionally upload and verify your key.</li>
+            <li>During key creation, you can optionally upload and verify your key.</li>
+            <li>Keys can be discovered by e-mail address.</li>
         </ul>
     </p>
 
-    <h2 style="margin-left: 3%;"><img src="/assets/img/openkeychain.svg" style="display: inline; height: 2em; vertical-align: -30%; padding-right: 10px;" /> OpenKeychain</h2>
+    <h2>
+        <div><img src="/assets/img/gpgtools.png"></div>
+        GPG Suite
+    </h2>
     <p>
-        OpenKeychain will support <span class="brand">keys.openpgp.org</span>
-        out of the box in an upcoming version. Stay tuned!
+        <a href="https://gpgtools.org/">GPG Suite</a> for macOS
+        uses <span class="brand">keys.openpgp.org</span> by default
+        since August 2019.
+    </p>
+
+    <h2>
+        <div><img src="/assets/img/openkeychain.svg"></div>
+        OpenKeychain
+    </h2>
+    <p>
+        <a href="https://www.openkeychain.org/">OpenKeychain</a> for Android
+        uses <span class="brand">keys.openpgp.org</span> by default
+        since July 2019.
         <ul>
             <li>Keys will be kept up to date automatically.</li>
-            <li>During creation, you can optionally upload and verify your key.</li>
+            <li>Keys can be discovered by e-mail address.</li>
         </ul>
     </p>
 
-    <h2 style="margin-left: 3%;"><img src="/assets/img/gnupg.svg" style="display: inline; height: 2em; vertical-align: -30%; padding-right: 10px;" /> GnuPG</h2>
+    <p>
+        Note that there is no built-in support for upload and e-mail verification so far.
+    </p>
 
-    To configure GnuPG to use <span class="brand">keys.openpgp.org</span>
-    as keyserver, add this line to your <tt>gpg.conf</tt> file:
-    <blockquote>
-        keyserver hkps://keys.openpgp.org
-    </blockquote>
+    <h2>
+        <div><img src="/assets/img/gnupg.svg" /></div>
+        GnuPG
+    </h2>
 
-    <h3>Retrieving keys</h3>
+    <p>
+        To configure <a href="https://gnupg.org">GnuPG</a>
+        to use <span class="brand">keys.openpgp.org</span> as keyserver,
+        add this line to your <tt>gpg.conf</tt> file:
+        <blockquote>
+            keyserver hkps://keys.openpgp.org
+        </blockquote>
+    </p>
+
+    <h4 id="gnupg-retrieve"><a href="#gnupg-retrieve">Retrieving keys</a></h4>
     <ul>
         <li>
             To locate the key of a user, by email address:
@@ -46,14 +78,10 @@
         </li>
         <li>To refresh all your keys (e.g. new revocation certificates and subkeys):
             <blockquote>gpg --refresh-keys</blockquote>
-            <b>Note:</b> If you see errors like the following,
-            see <a href="/about/faq#older-gnupg">our notes</a> on compatibility
-            with older versions of GnuPG.
-            <blockquote>gpg: key A2604867523C7ED8: no user ID</blockquote>
         </li>
     </ul>
 
-    <h3 id="gnupg-upload">Uploading your key</h3>
+    <h4 id="gnupg-upload"><a href="#gnupg-upload">Uploading your key</a></h4>
     <p>
         Keys can be uploaded with GnuPG's <tt>--send-keys</tt> command, but
         identity information can't be verified that way to make the key
@@ -77,7 +105,28 @@
         </li>
     </ul>
 
-    <h3>Usage via Tor</h3>
+    <h4 id="gnupg-troubleshooting"><a href="#gnupg-troubleshooting">Troubleshooting</a></h4>
+    <ul>
+        <li>
+            Some old <tt>~/gnupg/dirmngr.conf</tt> files contain a line like this:
+            <blockquote>
+                hkp-cacert ~/.gnupg/sks-keyservers.netCA.pem
+            </blockquote>
+            <p>
+                This configuration is no longer necessary,
+                but prevents regular certificates from working.
+                It is recommended to simply remove this line from the configuration.
+            </p>
+        </li>
+        <li>
+            While refreshing keys, you may see errors like the following:
+            <blockquote>gpg: key A2604867523C7ED8: no user ID</blockquote>
+            This is a <a href="https://dev.gnupg.org/T4393" target="_blank">known problem in GnuPG</a>.
+            We are working with the GnuPG team to resolve this issue.
+        </li>
+    </ul>
+
+    <h4 id="gnupg-tor"><a href="#gnupg-tor">Usage via Tor</a></h4>
     <p>
         For users who want to be extra-careful,
         <span class="brand">keys.openpgp.org</span> can be reached anonymously as an

dist/templates/email/welcome-html.hbs

@@ -0,0 +1,27 @@
+<!doctype html>
+<html lang=en>
+  <head>
+    <meta charset=utf-8>
+    <title>Your key upload on {{domain}}</title>
+  </head>
+  <body>
+    <p>
+      Hi,
+    <p>
+      this is an automated message from <a rel="nofollow" href="{{base_uri}}" style="text-decoration:none; color: #333"><tt>{{domain}}</tt></a>. If you didn't
+      upload your key there, please ignore this message.
+    <p>
+      OpenPGP key: <tt>{{primary_fp}}</tt>
+    <p>
+      This key was just uploaded for the first time, and is now published without
+      identity information. If you want to allow others to find this key by e-mail
+      address, please follow this link:
+    <p>
+      <a href="{{uri}}">{{uri}}</a>
+    <p>
+      You can find more info at <a href="{{base_uri}}/about">{{domain}}/about</a>.
+    <p>
+      Greetings from the <a rel="nofollow" href="{{base_uri}}" style="text-decoration:none; color: #333"><tt>keys.openpgp.org</tt></a> team
+  </body>
+</html>
+

dist/templates/email/welcome-txt.hbs

@@ -0,0 +1,16 @@
+Hi,
+
+this is an automated message from {{domain}}. If you didn't upload your key
+there, please ignore this message.
+
+OpenPGP key: {{primary_fp}}
+
+This key was just uploaded for the first time, and is now published without
+identity information. If you want to allow others to find this key by e-mail
+address, please follow this link:
+
+    {{uri}}
+
+You can find more info at {{base_uri}}/about
+
+Greetings from the keys.openpgp.org team

dist/templates/layout.html.hbs

@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-    <link rel="stylesheet" href="/assets/site.css?v=17" type="text/css"/>
+    <link rel="stylesheet" href="/assets/site.css?v=18" type="text/css"/>
     <title>keys.openpgp.org</title>
 </head>
   <body lang="en">

dist/templates/upload/upload.html.hbs

@@ -15,21 +15,6 @@
     Need more info? Check our <a target="_blank" href="/about">intro</a> and <a target="_blank" href="/about/usage">usage guide</a>!
   </p>
 
-  <script>
-    var body = document.getElementsByTagName("body")[0];
-
-     body.addEventListener('dragover', function(e) {
-        e.preventDefault();
-        e.stopPropagation();
-      }, false);
-
-      body.addEventListener('drop', function(e) {
-        e.preventDefault();
-        e.stopPropagation();
-        if (e.dataTransfer) {
-          document.getElementById('keytext').files = e.dataTransfer.files;
-        }
-    }, false);
-  </script>
+  <script src="/assets/js/upload.js" async />
 </div>
 {{/layout}}

src/anonymize_utils.rs

@@ -0,0 +1,90 @@
+use lazy_static::lazy_static;
+use std::collections::HashSet;
+
+use crate::database::types::Email;
+
+// from https://github.com/mailcheck/mailcheck/wiki/List-of-Popular-Domains
+lazy_static! {
+    static ref POPULAR_DOMAINS: HashSet<&'static str> = vec!(
+        /* Default domains included */
+        "aol.com", "att.net", "comcast.net", "facebook.com", "gmail.com", "gmx.com", "googlemail.com",
+        "google.com", "hotmail.com", "hotmail.co.uk", "mac.com", "me.com", "mail.com", "msn.com",
+        "live.com", "sbcglobal.net", "verizon.net", "yahoo.com", "yahoo.co.uk",
+
+        /* Other global domains */
+        "email.com", "fastmail.fm", "games.com" /* AOL */, "gmx.net", "hush.com", "hushmail.com", "icloud.com",
+        "iname.com", "inbox.com", "lavabit.com", "love.com" /* AOL */, "mailbox.org", "posteo.de", "outlook.com", "pobox.com", "protonmail.ch", "protonmail.com", "tutanota.de", "tutanota.com", "tutamail.com", "tuta.io",
+        "keemail.me", "rocketmail.com" /* Yahoo */, "safe-mail.net", "wow.com" /* AOL */, "ygm.com" /* AOL */,
+        "ymail.com" /* Yahoo */, "zoho.com", "yandex.com",
+
+        /* United States ISP domains */
+        "bellsouth.net", "charter.net", "cox.net", "earthlink.net", "juno.com",
+
+        /* British ISP domains */
+        "btinternet.com", "virginmedia.com", "blueyonder.co.uk", "freeserve.co.uk", "live.co.uk",
+        "ntlworld.com", "o2.co.uk", "orange.net", "sky.com", "talktalk.co.uk", "tiscali.co.uk",
+        "virgin.net", "wanadoo.co.uk", "bt.com",
+
+        /* Domains used in Asia */
+        "sina.com", "sina.cn", "qq.com", "naver.com", "hanmail.net", "daum.net", "nate.com", "yahoo.co.jp", "yahoo.co.kr", "yahoo.co.id", "yahoo.co.in", "yahoo.com.sg", "yahoo.com.ph", "163.com", "yeah.net", "126.com", "21cn.com", "aliyun.com", "foxmail.com",
+
+        /* French ISP domains */
+        "hotmail.fr", "live.fr", "laposte.net", "yahoo.fr", "wanadoo.fr", "orange.fr", "gmx.fr", "sfr.fr", "neuf.fr", "free.fr",
+
+        /* German ISP domains */
+        "gmx.de", "hotmail.de", "live.de", "online.de", "t-online.de" /* T-Mobile */, "web.de", "yahoo.de",
+
+        /* Italian ISP domains */
+        "libero.it", "virgilio.it", "hotmail.it", "aol.it", "tiscali.it", "alice.it", "live.it", "yahoo.it", "email.it", "tin.it", "poste.it", "teletu.it",
+
+        /* Russian ISP domains */
+        "mail.ru", "rambler.ru", "yandex.ru", "ya.ru", "list.ru",
+
+        /* Belgian ISP domains */
+        "hotmail.be", "live.be", "skynet.be", "voo.be", "tvcablenet.be", "telenet.be",
+
+        /* Argentinian ISP domains */
+        "hotmail.com.ar", "live.com.ar", "yahoo.com.ar", "fibertel.com.ar", "speedy.com.ar", "arnet.com.ar",
+
+        /* Domains used in Mexico */
+        "yahoo.com.mx", "live.com.mx", "hotmail.es", "hotmail.com.mx", "prodigy.net.mx",
+
+        /* Domains used in Brazil */
+        "yahoo.com.br", "hotmail.com.br", "outlook.com.br", "uol.com.br", "bol.com.br", "terra.com.br", "ig.com.br", "itelefonica.com.br", "r7.com", "zipmail.com.br", "globo.com", "globomail.com", "oi.com.br"
+    ).into_iter().collect();
+}
+
+pub fn anonymize_address(email: &Email) -> Option<String> {
+    email.as_str()
+      .rsplit('@')
+      .next()
+      .map(|domain| domain.to_lowercase())
+      .and_then(|domain| {
+        if POPULAR_DOMAINS.contains(&domain.as_str()) {
+            Some(domain)
+        } else {
+            domain.rsplit('.').next().map(|tld| tld.to_owned())
+        }
+      })
+}
+
+pub fn anonymize_address_fallback(email: &Email) -> String {
+    anonymize_address(email).unwrap_or_else(|| "unknown".to_owned())
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn known_domain() {
+        let email = "user@hotmail.be".parse::<Email>().unwrap();
+        assert_eq!("hotmail.be", anonymize_address(&email).unwrap());
+    }
+
+    #[test]
+    fn unknown_domain() {
+        let email = "user@example.org".parse::<Email>().unwrap();
+        assert_eq!("org", anonymize_address(&email).unwrap());
+    }
+}

src/counters.rs

@@ -0,0 +1,67 @@
+use lazy_static::lazy_static;
+use rocket_prometheus::prometheus;
+
+use crate::anonymize_utils;
+
+use crate::database::types::Email;
+
+lazy_static! {
+    static ref KEY_UPLOAD: LabelCounter =
+        LabelCounter::new("hagrid_key_upload", "Uploaded keys", &["result"]);
+
+    static ref MAIL_SENT: LabelCounter =
+        LabelCounter::new("hagrid_mail_sent", "Sent verification mails", &["type", "domain"]);
+
+    static ref KEY_ADDRESS_PUBLISHED: LabelCounter =
+        LabelCounter::new("hagrid_key_address_published", "Verified email addresses", &["domain"]);
+    static ref KEY_ADDRESS_UNPUBLISHED: LabelCounter =
+        LabelCounter::new("hagrid_key_address_unpublished", "Unpublished email addresses", &["domain"]);
+}
+
+pub fn register_counters(registry: &prometheus::Registry) {
+    KEY_UPLOAD.register(registry);
+
+    MAIL_SENT.register(registry);
+
+    KEY_ADDRESS_PUBLISHED.register(registry);
+    KEY_ADDRESS_UNPUBLISHED.register(registry);
+}
+
+pub fn inc_key_upload(upload_result: &str) {
+    KEY_UPLOAD.inc(&[upload_result]);
+}
+
+pub fn inc_mail_sent(mail_type: &str, email: &Email) {
+    let anonymized_adddress = anonymize_utils::anonymize_address_fallback(email);
+    MAIL_SENT.inc(&[mail_type, &anonymized_adddress]);
+}
+
+pub fn inc_address_published(email: &Email) {
+    let anonymized_adddress = anonymize_utils::anonymize_address_fallback(email);
+    KEY_ADDRESS_PUBLISHED.inc(&[&anonymized_adddress]);
+}
+
+pub fn inc_address_unpublished(email: &Email) {
+    let anonymized_adddress = anonymize_utils::anonymize_address_fallback(email);
+    KEY_ADDRESS_UNPUBLISHED.inc(&[&anonymized_adddress]);
+}
+
+struct LabelCounter {
+    prometheus_counter: prometheus::IntCounterVec,
+}
+
+impl LabelCounter {
+    fn new(name: &str, help: &str, labels: &[&str]) -> Self {
+        let opts = prometheus::Opts::new(name, help);
+        let prometheus_counter = prometheus::IntCounterVec::new(opts, labels).unwrap();
+        Self { prometheus_counter }
+    }
+
+    fn register(&self, registry: &prometheus::Registry) {
+        registry.register(Box::new(self.prometheus_counter.clone())).unwrap();
+    }
+
+    fn inc(&self, values: &[&str]) {
+        self.prometheus_counter.with_label_values(values).inc();
+    }
+}

src/delete.rs

@@ -9,7 +9,7 @@ extern crate structopt;
 use structopt::StructOpt;
 
 extern crate hagrid_database as database;
-use database::{Query, Database, KeyDatabase};
+use crate::database::{Query, Database, KeyDatabase};
 
 #[derive(Debug, StructOpt)]
 #[structopt(

src/mail.rs

@@ -7,9 +7,10 @@ use lettre_email::{Mailbox,EmailBuilder};
 use url;
 use serde::Serialize;
 use uuid::Uuid;
+use crate::counters;
 
-use database::types::Email;
-use Result;
+use crate::database::types::Email;
+use crate::Result;
 
 mod context {
     #[derive(Serialize, Clone)]
@@ -28,6 +29,14 @@ mod context {
         pub base_uri: String,
         pub domain: String,
     }
+
+    #[derive(Serialize, Clone)]
+    pub struct Welcome {
+        pub primary_fp: String,
+        pub uri: String,
+        pub base_uri: String,
+        pub domain: String,
+    }
 }
 
 pub struct Service {
@@ -82,6 +91,8 @@ impl Service {
             domain: self.domain.clone(),
         };
 
+        counters::inc_mail_sent("verify", userid);
+
         self.send(
             &vec![userid],
             &format!("Verify {} for your key on {}", userid, self.domain),
@@ -99,6 +110,8 @@ impl Service {
             domain: self.domain.clone(),
         };
 
+        counters::inc_mail_sent("manage", recipient);
+
         self.send(
             &[recipient],
             &format!("Manage your key on {}", self.domain),
@@ -107,6 +120,26 @@ impl Service {
         )
     }
 
+    pub fn send_welcome(&self, base_uri: &str, tpk_name: String, userid: &Email,
+                             token: &str)
+                             -> Result<()> {
+        let ctx = context::Welcome {
+            primary_fp: tpk_name,
+            uri: format!("{}/upload/{}", base_uri, token),
+            base_uri: base_uri.to_owned(),
+            domain: self.domain.clone(),
+        };
+
+        counters::inc_mail_sent("welcome", userid);
+
+        self.send(
+            &vec![userid],
+            &format!("Your key upload on {}", self.domain),
+            "welcome",
+            ctx,
+        )
+    }
+
     fn send<T>(&self, to: &[&Email], subject: &str, template: &str, ctx: T)
                -> Result<()>
         where T: Serialize + Clone,

src/main.rs

@@ -20,8 +20,10 @@ extern crate rocket_contrib;
 
 extern crate sequoia_openpgp;
 extern crate handlebars;
+extern crate lazy_static;
 extern crate lettre;
 extern crate lettre_email;
+extern crate rocket_prometheus;
 extern crate tempfile;
 extern crate uuid;
 
@@ -32,11 +34,13 @@ extern crate ring;
 
 extern crate hagrid_database as database;
 mod mail;
+mod anonymize_utils;
 mod web;
 mod tokens;
 mod sealed_state;
 mod rate_limiter;
 mod dump;
+mod counters;
 
 fn main() {
     if let Err(e) = web::serve() {

src/tokens.rs

@@ -1,8 +1,8 @@
-use sealed_state::SealedState;
+use crate::sealed_state::SealedState;
 
 use serde_json;
 use serde::{Serialize,de::DeserializeOwned};
-use Result;
+use crate::Result;
 
 pub trait StatelessSerializable : Serialize + DeserializeOwned {
 }

src/web/debug_web.rs

@@ -1,9 +1,9 @@
 use std::io;
 
-use dump::{self, Kind};
-use web::MyResponse;
+use crate::dump::{self, Kind};
+use crate::web::MyResponse;
 
-use database::{Database, KeyDatabase, Query};
+use crate::database::{Database, KeyDatabase, Query};
 
 #[get("/debug?<q>")]
 pub fn debug_info(

src/web/hkp.rs

@@ -6,15 +6,17 @@ use rocket::http::{ContentType, Status};
 use rocket::request::{self, Request, FromRequest};
 use rocket::http::uri::Uri;
 
-use database::{Database, Query, KeyDatabase};
-use database::types::{Email, Fingerprint, KeyID};
+use crate::database::{Database, Query, KeyDatabase};
+use crate::database::types::{Email, Fingerprint, KeyID};
 
-use rate_limiter::RateLimiter;
+use crate::rate_limiter::RateLimiter;
 
-use tokens;
+use crate::tokens;
 
-use web;
-use web::{HagridState, RequestOrigin, MyResponse, vks_web};
+use crate::web;
+use crate::mail;
+use crate::web::{HagridState, RequestOrigin, MyResponse, vks_web};
+use crate::web::vks::response::UploadResponse;
 
 #[derive(Debug)]
 pub enum Hkp {
@@ -116,7 +118,7 @@ pub fn pks_add_form_data(
     cont_type: &ContentType,
     data: Data,
 ) -> MyResponse {
-    match vks_web::upload_post_form_data(db, tokens_stateless, rate_limiter, cont_type, data) {
+    match vks_web::process_post_form_data(db, tokens_stateless, rate_limiter, cont_type, data) {
         Ok(_) => MyResponse::plain("Ok".into()),
         Err(err) => MyResponse::ise(err),
     }
@@ -128,17 +130,41 @@ pub fn pks_add_form(
     db: rocket::State<KeyDatabase>,
     tokens_stateless: rocket::State<tokens::Service>,
     rate_limiter: rocket::State<RateLimiter>,
+    mail_service: rocket::State<mail::Service>,
     data: Data,
 ) -> MyResponse {
-    match vks_web::upload_post_form(db, tokens_stateless, rate_limiter, data) {
+    match vks_web::process_post_form(db, tokens_stateless, rate_limiter, data) {
+        Ok(UploadResponse::Ok { is_new_key, key_fpr, primary_uid, token, .. }) => {
+            let msg = if is_new_key && send_welcome_mail(&request_origin, &mail_service, key_fpr, primary_uid, token) {
+                "Upload successful. This is a new key, a welcome mail has been sent!".to_owned()
+            } else {
+                format!("Upload successful. Note that identity information will only be published after verification! see {}/about/usage#gnupg-upload", request_origin.get_base_uri())
+            };
+            MyResponse::plain(msg)
+        }
         Ok(_) => {
-            let msg = format!("Upload successful. Note that identity information will only be published with verification! see {}/about/usage#gnupg-upload", request_origin.get_base_uri());
+            let msg = format!("Upload successful. Note that identity information will only be published after verification! see {}/about/usage#gnupg-upload", request_origin.get_base_uri());
             MyResponse::plain(msg)
         }
         Err(err) => MyResponse::ise(err),
     }
 }
 
+fn send_welcome_mail(
+    request_origin: &RequestOrigin,
+    mail_service: &mail::Service,
+    fpr: String,
+    primary_uid: Option<Email>,
+    token: String,
+) -> bool {
+    if let Some(primary_uid) = primary_uid {
+        mail_service.send_welcome(
+            request_origin.get_base_uri(), fpr, &primary_uid, &token).is_ok()
+    } else {
+        false
+    }
+}
+
 #[get("/pks/lookup")]
 pub fn pks_lookup(state: rocket::State<HagridState>,
                   db: rocket::State<KeyDatabase>,
@@ -269,7 +295,7 @@ mod tests {
     use sequoia_openpgp::tpk::TPKBuilder;
     use sequoia_openpgp::serialize::Serialize;
 
-    use web::tests::*;
+    use crate::web::tests::*;
 
     #[test]
     fn hkp() {
@@ -306,9 +332,9 @@ mod tests {
         let body = response.body_string().unwrap();
         eprintln!("response: {}", body);
 
-        // Check that we do not get a confirmation mail.
-        let confirm_mail = pop_mail(filemail_into.as_path()).unwrap();
-        assert!(confirm_mail.is_none());
+        // Check that we get a welcome mail
+        let welcome_mail = pop_mail(filemail_into.as_path()).unwrap();
+        assert!(welcome_mail.is_some());
 
         // We should not be able to look it up by email address.
         check_null_responses_by_email(&client, "foo@invalid.example.com");
@@ -320,6 +346,16 @@ mod tests {
         // And check that we can see the human-readable result page.
         check_hr_responses_by_fingerprint(&client, &tpk, 0);
 
+        // Upload the same key again, make sure the welcome mail is not sent again
+        let response = client.post("/pks/add")
+            .body(post_data.as_bytes())
+            .header(ContentType::Form)
+            .dispatch();
+        assert_eq!(response.status(), Status::Ok);
+
+        let welcome_mail = pop_mail(filemail_into.as_path()).unwrap();
+        assert!(welcome_mail.is_none());
+
         assert_consistency(client.rocket());
     }
 
@@ -356,8 +392,11 @@ mod tests {
             .header(ContentType::Form)
             .dispatch();
         assert_eq!(response.status(), Status::Ok);
-        let confirm_mail = pop_mail(filemail_into.as_path()).unwrap();
-        assert!(confirm_mail.is_none());
+
+        // Check that there is no welcome mail (since we uploaded two)
+        let welcome_mail = pop_mail(filemail_into.as_path()).unwrap();
+        assert!(welcome_mail.is_none());
+
         check_mr_responses_by_fingerprint(&client, &tpk_0, 0);
         check_mr_responses_by_fingerprint(&client, &tpk_1, 0);
         check_hr_responses_by_fingerprint(&client, &tpk_0, 0);

src/web/maintenance.rs

@@ -6,7 +6,7 @@ use rocket::http::Method;
 use std::fs;
 use std::path::PathBuf;
 
-use web::MyResponse;
+use crate::web::MyResponse;
 
 pub struct MaintenanceMode {
     maintenance_file: PathBuf,

src/web/manage.rs

@@ -4,12 +4,13 @@ use rocket::request::Form;
 
 use failure::Fallible as Result;
 
-use web::{RequestOrigin, MyResponse, templates::General};
-use web::vks_web;
-use database::{Database, KeyDatabase, types::Email, types::Fingerprint};
-use mail;
-use rate_limiter::RateLimiter;
-use tokens::{self, StatelessSerializable};
+use crate::web::{RequestOrigin, MyResponse, templates::General};
+use crate::web::vks_web;
+use crate::database::{Database, KeyDatabase, types::Email, types::Fingerprint};
+use crate::mail;
+use crate::counters;
+use crate::rate_limiter::RateLimiter;
+use crate::tokens::{self, StatelessSerializable};
 
 #[derive(Debug,Serialize,Deserialize)]
 struct StatelessVerifyToken {
@@ -67,7 +68,7 @@ pub fn vks_manage_key(
    token: String,
    token_service: rocket::State<tokens::Service>,
 ) -> MyResponse {
-    use database::types::Fingerprint;
+    use crate::database::types::Fingerprint;
     use std::convert::TryFrom;
     if let Ok(StatelessVerifyToken { fpr }) = token_service.check(&token) {
         match db.lookup(&database::Query::ByFingerprint(fpr)) {
@@ -186,6 +187,9 @@ pub fn vks_manage_unpublish_or_fail(
 ) -> Result<MyResponse> {
     let verify_token = token_service.check::<StatelessVerifyToken>(&request.token)?;
     let email = request.address.parse::<Email>()?;
+
     db.set_email_unpublished(&verify_token.fpr, &email)?;
+    counters::inc_address_unpublished(&email);
+
     Ok(vks_manage_key(request_origin, db, request.token.to_owned(), token_service))
 }

src/web/mod.rs

@@ -9,18 +9,21 @@ use rocket::http::uri::Uri;
 use rocket_contrib::json::JsonValue;
 use rocket::response::status::Custom;
 
+use rocket_prometheus::PrometheusMetrics;
+
 use serde::Serialize;
 use handlebars::Handlebars;
 
 use std::path::PathBuf;
 
-use mail;
-use tokens;
-use rate_limiter::RateLimiter;
+use crate::mail;
+use crate::tokens;
+use crate::counters;
+use crate::rate_limiter::RateLimiter;
 
-use database::{Database, KeyDatabase, Query};
-use database::types::Fingerprint;
-use Result;
+use crate::database::{Database, KeyDatabase, Query};
+use crate::database::types::Fingerprint;
+use crate::Result;
 
 use std::convert::TryInto;
 
@@ -32,7 +35,7 @@ mod vks_web;
 mod vks_api;
 mod debug_web;
 
-use web::maintenance::MaintenanceMode;
+use crate::web::maintenance::MaintenanceMode;
 
 use rocket::http::hyper::header::ContentDisposition;
 
@@ -332,7 +335,7 @@ pub fn serve() -> Result<()> {
     Err(rocket_factory(rocket::ignite())?.launch().into())
 }
 
-fn rocket_factory(rocket: rocket::Rocket) -> Result<rocket::Rocket> {
+fn rocket_factory(mut rocket: rocket::Rocket) -> Result<rocket::Rocket> {
     let routes = routes![
         // infra
         root,
@@ -389,7 +392,9 @@ fn rocket_factory(rocket: rocket::Rocket) -> Result<rocket::Rocket> {
     let rate_limiter = configure_rate_limiter(rocket.config())?;
     let maintenance_mode = configure_maintenance_mode(rocket.config())?;
 
-    Ok(rocket
+    let prometheus = configure_prometheus(rocket.config());
+
+    rocket = rocket
        .attach(Template::fairing())
        .attach(maintenance_mode)
        .manage(hagrid_state)
@@ -398,8 +403,24 @@ fn rocket_factory(rocket: rocket::Rocket) -> Result<rocket::Rocket> {
        .manage(mail_service)
        .manage(db_service)
        .manage(rate_limiter)
-       .mount("/", routes)
-      )
+       .mount("/", routes);
+
+    if let Some(prometheus) = prometheus {
+        rocket = rocket
+            .attach(prometheus.clone())
+            .mount("/metrics", prometheus);
+    }
+
+    Ok(rocket)
+}
+
+fn configure_prometheus(config: &Config) -> Option<PrometheusMetrics> {
+    if !config.get_bool("enable_prometheus").unwrap_or(false) {
+        return None;
+    }
+    let prometheus = PrometheusMetrics::new();
+    counters::register_counters(&prometheus.registry());
+    return Some(prometheus);
 }
 
 fn configure_db_service(config: &Config) -> Result<KeyDatabase> {
@@ -455,12 +476,16 @@ fn configure_mail_service(config: &Config) -> Result<mail::Service> {
     let verify_txt = template_dir.join("email/publish-txt.hbs");
     let manage_html = template_dir.join("email/manage-html.hbs");
     let manage_txt = template_dir.join("email/manage-txt.hbs");
+    let welcome_html = template_dir.join("email/welcome-html.hbs");
+    let welcome_txt = template_dir.join("email/welcome-txt.hbs");
 
     let mut handlebars = Handlebars::new();
     handlebars.register_template_file("verify-html", verify_html)?;
     handlebars.register_template_file("verify-txt", verify_txt)?;
     handlebars.register_template_file("manage-html", manage_html)?;
     handlebars.register_template_file("manage-txt", manage_txt)?;
+    handlebars.register_template_file("welcome-html", welcome_html)?;
+    handlebars.register_template_file("welcome-txt", welcome_txt)?;
 
     let filemail_into = config.get_str("filemail_into")
         .ok().map(|p| PathBuf::from(p));
@@ -503,7 +528,7 @@ pub mod tests {
     use sequoia_openpgp::parse::Parse;
     use sequoia_openpgp::serialize::Serialize;
 
-    use database::*;
+    use crate::database::*;
     use super::*;
 
     // for some reason, this is no longer public in lettre itself
@@ -528,7 +553,7 @@ pub mod tests {
     /// duration of your test.  To debug the test, mem::forget it to
     /// prevent cleanup.
     pub fn configuration() -> Result<(TempDir, rocket::Config)> {
-        use rocket::config::{Config, Environment};
+        use rocket::config::Environment;
 
         let root = tempdir()?;
         let filemail = root.path().join("filemail");
@@ -1000,7 +1025,7 @@ pub mod tests {
     /// Asserts that the given URI returns human readable response
     /// page that contains an onion URI pointing to the TPK.
     pub fn check_hr_response_onion(client: &Client, uri: &str, tpk: &TPK,
-                             nr_uids: usize) {
+                             _nr_uids: usize) {
         let mut response = client
             .get(uri)
             .header(Header::new("X-Is-Onion", "true"))

src/web/vks.rs

@@ -1,11 +1,12 @@
 use failure::Fallible as Result;
 
-use database::{Database, KeyDatabase, StatefulTokens, EmailAddressStatus, TpkStatus};
-use database::types::{Fingerprint,Email};
-use mail;
-use tokens::{self, StatelessSerializable};
-use rate_limiter::RateLimiter;
-use web::RequestOrigin;
+use crate::database::{Database, KeyDatabase, StatefulTokens, EmailAddressStatus, TpkStatus, ImportResult};
+use crate::database::types::{Fingerprint,Email};
+use crate::mail;
+use crate::counters;
+use crate::tokens::{self, StatelessSerializable};
+use crate::rate_limiter::RateLimiter;
+use crate::web::RequestOrigin;
 
 use sequoia_openpgp::TPK;
 
@@ -29,6 +30,8 @@ pub mod request {
 }
 
 pub mod response {
+    use crate::database::types::Email;
+
     #[derive(Debug,Serialize,Deserialize,PartialEq,Eq)]
     pub enum EmailStatus {
         #[serde(rename = "unpublished")]
@@ -50,6 +53,8 @@ pub mod response {
             is_revoked: bool,
             status: HashMap<String,EmailStatus>,
             count_unparsed: usize,
+            is_new_key: bool,
+            primary_uid: Option<Email>,
         },
         OkMulti { key_fprs: Vec<String> },
         Error(String),
@@ -107,6 +112,7 @@ pub fn process_key(
         tpks.push(match tpk {
             Ok(t) => {
                 if t.is_tsk() {
+                    counters::inc_key_upload("secret");
                     return UploadResponse::err("Whoops, please don't upload secret keys!");
                 }
                 t
@@ -122,6 +128,17 @@ pub fn process_key(
     }
 }
 
+fn log_db_merge(import_result: Result<ImportResult>) -> Result<ImportResult> {
+    match import_result {
+        Ok(ImportResult::New(_)) => counters::inc_key_upload("new"),
+        Ok(ImportResult::Updated(_)) => counters::inc_key_upload("updated"),
+        Ok(ImportResult::Unchanged(_)) => counters::inc_key_upload("unchanged"),
+        Err(_) => counters::inc_key_upload("error"),
+    };
+
+    import_result
+}
+
 fn process_key_multiple(
     db: &KeyDatabase,
     tpks: Vec<TPK>,
@@ -130,7 +147,7 @@ fn process_key_multiple(
         .into_iter()
         .flat_map(|tpk| Fingerprint::try_from(tpk.fingerprint())
                 .map(|fpr| (fpr, tpk)))
-        .flat_map(|(fpr, tpk)| db.merge(tpk).map(|_| fpr.to_string()))
+        .flat_map(|(fpr, tpk)| log_db_merge(db.merge(tpk)).map(|_| fpr.to_string()))
         .collect();
 
     response::UploadResponse::OkMulti { key_fprs }
@@ -144,8 +161,10 @@ fn process_key_single(
 ) -> response::UploadResponse {
     let fp = Fingerprint::try_from(tpk.fingerprint()).unwrap();
 
-    let tpk_status = match db.merge(tpk) {
-        Ok(import_result) => import_result.into_tpk_status(),
+    let (tpk_status, is_new_key) = match log_db_merge(db.merge(tpk)) {
+        Ok(ImportResult::New(tpk_status)) => (tpk_status, true),
+        Ok(ImportResult::Updated(tpk_status)) => (tpk_status, false),
+        Ok(ImportResult::Unchanged(tpk_status)) => (tpk_status, false),
         Err(_) => return UploadResponse::err(&format!(
             "Something went wrong processing key {}", fp)),
     };
@@ -163,7 +182,7 @@ fn process_key_single(
 
     let token = tokens_stateless.create(&verify_state);
 
-    show_upload_verify(rate_limiter, token, tpk_status, verify_state)
+    show_upload_verify(rate_limiter, token, tpk_status, verify_state, is_new_key)
 }
 
 pub fn request_verify(
@@ -183,7 +202,7 @@ pub fn request_verify(
 
     if tpk_status.is_revoked {
         return show_upload_verify(
-                &rate_limiter, token, tpk_status, verify_state);
+                &rate_limiter, token, tpk_status, verify_state, false);
     }
 
     let emails_requested: Vec<_> = addresses.into_iter()
@@ -205,7 +224,7 @@ pub fn request_verify(
         }
     }
 
-    show_upload_verify(&rate_limiter, token, tpk_status, verify_state)
+    show_upload_verify(&rate_limiter, token, tpk_status, verify_state, false)
 }
 
 fn check_tpk_state(
@@ -260,7 +279,9 @@ fn check_publish_token(
 ) -> Result<(Fingerprint,Email)> {
     let payload = token_service.pop_token("verify", &token)?;
     let (fingerprint, email) = serde_json::from_str(&payload)?;
+
     db.set_email_published(&fingerprint, &email)?;
+    counters::inc_address_published(&email);
 
     Ok((fingerprint, email))
 }
@@ -270,10 +291,19 @@ fn show_upload_verify(
     token: String,
     tpk_status: TpkStatus,
     verify_state: VerifyTpkState,
+    is_new_key: bool,
 ) -> response::UploadResponse {
     let key_fpr = verify_state.fpr.to_string();
     if tpk_status.is_revoked {
-        return response::UploadResponse::Ok { token, key_fpr, count_unparsed: 0, is_revoked: true, status: HashMap::new() };
+        return response::UploadResponse::Ok {
+            token,
+            key_fpr,
+            count_unparsed: 0,
+            is_revoked: true,
+            status: HashMap::new(),
+            is_new_key: false,
+            primary_uid: None,
+        };
     }
 
     let status: HashMap<_,_> = tpk_status.email_status
@@ -292,8 +322,12 @@ fn show_upload_verify(
             }
         })
         .collect();
+    let primary_uid = tpk_status.email_status
+        .get(0)
+        .map(|(email, _)| email)
+        .cloned();
 
     let count_unparsed = tpk_status.unparsed_uids;
 
-    response::UploadResponse::Ok { token, key_fpr, count_unparsed, is_revoked: false, status }
+    response::UploadResponse::Ok { token, key_fpr, count_unparsed, is_revoked: false, status, is_new_key, primary_uid }
 }

src/web/vks_api.rs

@@ -4,19 +4,19 @@ use rocket::response::{self, Response, Responder};
 use rocket::http::{ContentType,Status};
 use std::io::Cursor;
 
-use database::{KeyDatabase, StatefulTokens, Query};
-use database::types::{Email, Fingerprint, KeyID};
-use mail;
-use tokens;
-use rate_limiter::RateLimiter;
+use crate::database::{KeyDatabase, StatefulTokens, Query};
+use crate::database::types::{Email, Fingerprint, KeyID};
+use crate::mail;
+use crate::tokens;
+use crate::rate_limiter::RateLimiter;
 
-use web;
-use web::{HagridState, RequestOrigin, MyResponse};
-use web::vks;
-use web::vks::response::*;
+use crate::web;
+use crate::web::{HagridState, RequestOrigin, MyResponse};
+use crate::web::vks;
+use crate::web::vks::response::*;
 
 pub mod json {
-    use web::vks::response::EmailStatus;
+    use crate::web::vks::response::EmailStatus;
     use std::collections::HashMap;
 
     #[derive(Deserialize)]

src/web/vks_web.rs

@@ -9,17 +9,17 @@ use rocket::http::ContentType;
 use rocket::request::Form;
 use rocket::Data;
 
-use database::{KeyDatabase, StatefulTokens, Query, Database};
-use mail;
-use tokens;
-use web::{RequestOrigin, MyResponse};
-use rate_limiter::RateLimiter;
+use crate::database::{KeyDatabase, StatefulTokens, Query, Database};
+use crate::mail;
+use crate::tokens;
+use crate::web::{RequestOrigin, MyResponse};
+use crate::rate_limiter::RateLimiter;
 
 use std::io::Read;
 use std::collections::HashMap;
 
-use web::vks;
-use web::vks::response::*;
+use crate::web::vks;
+use crate::web::vks::response::*;
 
 const UPLOAD_LIMIT: u64 = 1024 * 1024; // 1 MiB.
 
@@ -120,7 +120,7 @@ impl MyResponse {
 
     fn upload_response(response: UploadResponse) -> Self {
         match response {
-            UploadResponse::Ok { token, key_fpr, is_revoked, count_unparsed, status } =>
+            UploadResponse::Ok { token, key_fpr, is_revoked, count_unparsed, status, .. } =>
                 Self::upload_ok(token, key_fpr, is_revoked, count_unparsed, status),
             UploadResponse::OkMulti { key_fprs } =>
                 Self::upload_ok_multi(key_fprs),
@@ -215,16 +215,26 @@ pub fn upload_post_form_data(
     rate_limiter: rocket::State<RateLimiter>,
     cont_type: &ContentType,
     data: Data,
-) -> Result<MyResponse> {
+) -> MyResponse {
+    match process_post_form_data(db, tokens_stateless, rate_limiter, cont_type, data) {
+        Ok(response) => MyResponse::upload_response(response),
+        Err(err) => MyResponse::bad_request("upload/upload", err),
+    }
+}
+
+pub fn process_post_form_data(
+    db: rocket::State<KeyDatabase>,
+    tokens_stateless: rocket::State<tokens::Service>,
+    rate_limiter: rocket::State<RateLimiter>,
+    cont_type: &ContentType,
+    data: Data,
+) -> Result<UploadResponse> {
     // multipart/form-data
-    let (_, boundary) =
-        match cont_type.params().find(|&(k, _)| k == "boundary") {
-            Some(v) => v,
-            None => return Ok(MyResponse::bad_request(
-                "upload/upload",
-                failure::err_msg("`Content-Type: multipart/form-data` \
-                                    boundary param not provided"))),
-        };
+    let (_, boundary) = cont_type
+        .params()
+        .find(|&(k, _)| k == "boundary")
+        .ok_or_else(|| failure::err_msg("`Content-Type: multipart/form-data` \
+                                      boundary param not provided"))?;
 
     process_upload(&db, &tokens_stateless, &rate_limiter, data, boundary)
 }
@@ -310,7 +320,19 @@ pub fn upload_post_form(
     tokens_stateless: rocket::State<tokens::Service>,
     rate_limiter: rocket::State<RateLimiter>,
     data: Data,
-) -> Result<MyResponse> {
+) -> MyResponse {
+    match process_post_form(db, tokens_stateless, rate_limiter, data) {
+        Ok(response) => MyResponse::upload_response(response),
+        Err(err) => MyResponse::bad_request("upload/upload", err),
+    }
+}
+
+pub fn process_post_form(
+    db: rocket::State<KeyDatabase>,
+    tokens_stateless: rocket::State<tokens::Service>,
+    rate_limiter: rocket::State<RateLimiter>,
+    data: Data,
+) -> Result<UploadResponse> {
     use rocket::request::FormItems;
     use std::io::Cursor;
 
@@ -329,19 +351,18 @@ pub fn upload_post_form(
 
         match key.as_str() {
             "keytext" => {
-                return Ok(MyResponse::upload_response(vks::process_key(
+                return Ok(vks::process_key(
                     &db,
                     &tokens_stateless,
                     &rate_limiter,
                     Cursor::new(decoded_value.as_bytes())
-                )));
+                ));
             }
             _ => { /* skip */ }
         }
     }
 
-    Ok(MyResponse::bad_request("upload/upload",
-                                failure::err_msg("No keytext found")))
+    Err(failure::err_msg("No keytext found"))
 }
 
 
@@ -351,7 +372,7 @@ fn process_upload(
     rate_limiter: &RateLimiter,
     data: Data,
     boundary: &str,
-) -> Result<MyResponse> {
+) -> Result<UploadResponse> {
     // saves all fields, any field longer than 10kB goes to a temporary directory
     // Entries could implement FromData though that would give zero control over
     // how the files are saved; Multipart would be a good impl candidate though
@@ -371,18 +392,14 @@ fn process_multipart(
     tokens_stateless: &tokens::Service,
     rate_limiter: &RateLimiter,
     entries: Entries,
-) -> Result<MyResponse> {
+) -> Result<UploadResponse> {
     match entries.fields.get("keytext") {
         Some(ent) if ent.len() == 1 => {
             let reader = ent[0].data.readable()?;
-            Ok(MyResponse::upload_response(vks::process_key(db, tokens_stateless, rate_limiter, reader)))
+            Ok(vks::process_key(db, tokens_stateless, rate_limiter, reader))
         }
-        Some(_) =>
-            Ok(MyResponse::bad_request(
-                "upload/upload", failure::err_msg("Multiple keytexts found"))),
-        None =>
-            Ok(MyResponse::bad_request(
-                "upload/upload", failure::err_msg("No keytext found"))),
+        Some(_) => Err(failure::err_msg("Multiple keytexts found")),
+        None => Err(failure::err_msg("No keytext found")),
     }
 }