diff options
| -rw-r--r-- | rcynic/rcynic.c | 379 | ||||
| -rw-r--r-- | rcynic/rcynic.py | 53 | ||||
| -rw-r--r-- | rpkid/examples/rpki.conf | 4 | ||||
| -rw-r--r-- | rpkid/irbe_cli.py | 7 | ||||
| -rw-r--r-- | rpkid/rpki/__doc__.py | 2 | ||||
| -rw-r--r-- | rpkid/rpki/rootd.py | 2 | ||||
| -rw-r--r-- | rpkid/rpki/rpkid.py | 2 | ||||
| -rw-r--r-- | rpkid/rpki/up_down.py | 2 | ||||
| -rw-r--r-- | rpkid/rpki/x509.py | 52 | ||||
| -rw-r--r-- | rpkid/tests/Makefile.in | 2 | ||||
| -rw-r--r-- | rpkid/tests/left-right-protocol-samples.xml | 2 | ||||
| -rw-r--r-- | rpkid/tests/publication-protocol-samples.xml | 8 | ||||
| -rw-r--r-- | rpkid/tests/smoketest.3.yaml | 4 | ||||
| -rw-r--r-- | rpkid/tests/smoketest.py | 4 | ||||
| -rw-r--r-- | rpkid/tests/testpoke.py | 2 | ||||
| -rw-r--r-- | scripts/analyze-rcynic-history.py | 223 |
16 files changed, 392 insertions, 356 deletions
diff --git a/rcynic/rcynic.c b/rcynic/rcynic.c index 3dc3c044..d345dc9f 100644 --- a/rcynic/rcynic.c +++ b/rcynic/rcynic.c @@ -219,6 +219,7 @@ static const struct { QB(bad_key_usage, "Bad keyUsage") \ QB(bad_manifest_digest_length, "Bad manifest digest length") \ QB(bad_public_key, "Bad public key") \ + QB(bad_roa_asID, "Bad ROA asID") \ QB(bad_serial_number, "Bad serialNumber") \ QB(certificate_bad_signature, "Bad certificate signature") \ QB(certificate_failed_validation, "Certificate failed validation") \ @@ -253,6 +254,7 @@ static const struct { QB(nonconformant_signature_algorithm, "Nonconformant signature algorithm")\ QB(nonconformant_digest_algorithm, "Nonconformant digest algorithm") \ QB(object_rejected, "Object rejected") \ + QB(rfc3779_inheritance_required, "RFC 3779 inheritance required") \ QB(roa_contains_bad_afi_value, "ROA contains bad AFI value") \ QB(roa_resource_not_in_ee, "ROA resource not in EE") \ QB(roa_resources_malformed, "ROA resources malformed") \ @@ -272,6 +274,7 @@ static const struct { QW(crldp_names_newer_crl, "CRLDP names newer CRL") \ QW(digest_mismatch, "Digest mismatch") \ QW(issuer_uses_multiple_crldp_values, "Issuer uses multiple CRLDP values")\ + QW(multiple_rsync_uris_in_extension, "Multiple rsync URIs in extension") \ QW(nonconformant_issuer_name, "Nonconformant X.509 issuer name") \ QW(nonconformant_subject_name, "Nonconformant X.509 subject name") \ QW(rsync_transfer_skipped, "rsync transfer skipped") \ @@ -2781,7 +2784,7 @@ static int extract_crldp_uri(rcynic_ctx_t *rc, DIST_POINT *d; int i; - assert(crldp); + assert(rc && uri && crldp && result); if (sk_DIST_POINT_num(crldp) != 1) goto bad; @@ -2795,16 +2798,18 @@ static int extract_crldp_uri(rcynic_ctx_t *rc, GENERAL_NAME *n = sk_GENERAL_NAME_value(d->distpoint->name.fullname, i); if (n == NULL || n->type != GEN_URI) goto bad; - if (!is_rsync((char *) n->d.uniformResourceIdentifier->data)) { + if (!is_rsync((char *) n->d.uniformResourceIdentifier->data)) log_validation_status(rc, uri, non_rsync_uri_in_extension, generation); - } else if (sizeof(result->s) <= n->d.uniformResourceIdentifier->length) { + else if (sizeof(result->s) <= n->d.uniformResourceIdentifier->length) log_validation_status(rc, uri, uri_too_long, generation); - } else { + else if (result->s[0]) + log_validation_status(rc, uri, multiple_rsync_uris_in_extension, generation); + else strcpy(result->s, (char *) n->d.uniformResourceIdentifier->data); - return 1; - } } + return result->s[0]; + bad: log_validation_status(rc, uri, malformed_crldp_extension, generation); return 0; @@ -2819,11 +2824,12 @@ static int extract_access_uri(rcynic_ctx_t *rc, const AUTHORITY_INFO_ACCESS *xia, const unsigned char *oid, const int oidlen, - uri_t *result) + uri_t *result, + int *count) { int i; - assert(xia); + assert(rc && uri && xia && oid && result && count); for (i = 0; i < sk_ACCESS_DESCRIPTION_num(xia); i++) { ACCESS_DESCRIPTION *a = sk_ACCESS_DESCRIPTION_value(xia, i); @@ -2831,14 +2837,15 @@ static int extract_access_uri(rcynic_ctx_t *rc, return 0; if (oid_cmp(a->method, oid, oidlen)) continue; - if (!is_rsync((char *) a->location->d.uniformResourceIdentifier->data)) { + ++*count; + if (!is_rsync((char *) a->location->d.uniformResourceIdentifier->data)) log_validation_status(rc, uri, non_rsync_uri_in_extension, generation); - } else if (sizeof(result->s) <= a->location->d.uniformResourceIdentifier->length) { + else if (sizeof(result->s) <= a->location->d.uniformResourceIdentifier->length) log_validation_status(rc, uri, uri_too_long, generation); - } else { - strcpy(result->s, (char *) a->location->d.uniformResourceIdentifier->data); - return 1; - } + else if (result->s[0]) + log_validation_status(rc, uri, multiple_rsync_uris_in_extension, generation); + else + strcpy(result->s, (char *) a->location->d.uniformResourceIdentifier->data); } return 1; } @@ -3014,21 +3021,6 @@ static X509_CRL *check_crl_1(rcynic_ctx_t *rc, } } -#if 0 - /* - * Might need to generalize this to check cert AKI as well. Haven't - * handled cert SKI check yet either. Do we want to call - * X509_check_akid() here or just compare the OCTET STRINGs - * directly? 99% of X509_check_akid() is irrelevant to our profile. - */ - if (!crl->akid || - !crl->akid->keyid || - crl->akid->serial || - crl->akid->issuer || - X509_check_akid(issuer, crl->akid) != X509_V_OK) - bad_crl_akid; -#endif - if ((pkey = X509_get_pubkey(issuer)) == NULL) goto punt; ret = X509_CRL_verify(crl, pkey); @@ -3227,11 +3219,9 @@ static int check_x509(rcynic_ctx_t *rc, ASN1_BIT_STRING *ski_pubkey = NULL; STACK_OF(DIST_POINT) *crldp = NULL; BASIC_CONSTRAINTS *bc = NULL; - ASIdentifiers *asid = NULL; - IPAddrBlocks *addr = NULL; hashbuf_t ski_hashbuf; unsigned ski_hashlen; - int ok, crit, ex_count, ret = 0; + int ok, crit, loc, ex_count, ret = 0; assert(rc && wsk && w && uri && x && w->cert); @@ -3297,10 +3287,13 @@ static int check_x509(rcynic_ctx_t *rc, } if ((aia = X509_get_ext_d2i(x, NID_info_access, NULL, NULL)) != NULL) { + int n_caIssuers = 0; ex_count--; if (!extract_access_uri(rc, uri, generation, aia, - id_ad_caIssuers, sizeof(id_ad_caIssuers), &certinfo->aia) || - !certinfo->aia.s[0]) { + id_ad_caIssuers, sizeof(id_ad_caIssuers), + &certinfo->aia, &n_caIssuers) || + !certinfo->aia.s[0] || + sk_ACCESS_DESCRIPTION_num(aia) != n_caIssuers) { log_validation_status(rc, uri, malformed_aia_extension, generation); goto done; } @@ -3317,18 +3310,19 @@ static int check_x509(rcynic_ctx_t *rc, } if ((sia = X509_get_ext_d2i(x, NID_sinfo_access, NULL, NULL)) != NULL) { - int got_caDirectory, got_rpkiManifest, got_signedObject; + int got_caDirectory, got_rpkiManifest, got_signedObject; + int n_caDirectory = 0, n_rpkiManifest = 0, n_signedObject = 0; ex_count--; ok = (extract_access_uri(rc, uri, generation, sia, id_ad_caRepository, - sizeof(id_ad_caRepository), &certinfo->sia) && + sizeof(id_ad_caRepository), &certinfo->sia, &n_caDirectory) && extract_access_uri(rc, uri, generation, sia, id_ad_rpkiManifest, - sizeof(id_ad_rpkiManifest), &certinfo->manifest) && + sizeof(id_ad_rpkiManifest), &certinfo->manifest, &n_rpkiManifest) && extract_access_uri(rc, uri, generation, sia, id_ad_signedObject, - sizeof(id_ad_signedObject), &certinfo->signedobject)); + sizeof(id_ad_signedObject), &certinfo->signedobject, &n_signedObject)); got_caDirectory = certinfo->sia.s[0] != '\0'; got_rpkiManifest = certinfo->manifest.s[0] != '\0'; got_signedObject = certinfo->signedobject.s[0] != '\0'; - ok &= sk_ACCESS_DESCRIPTION_num(sia) == got_caDirectory + got_rpkiManifest + got_signedObject; + ok &= sk_ACCESS_DESCRIPTION_num(sia) == n_caDirectory + n_rpkiManifest + n_signedObject; if (certinfo->ca) ok &= got_caDirectory && got_rpkiManifest && !got_signedObject; else if (rc->allow_ee_without_signedObject) @@ -3435,23 +3429,28 @@ static int check_x509(rcynic_ctx_t *rc, } } - if ((addr = X509_get_ext_d2i(x, NID_sbgp_ipAddrBlock, &crit, NULL)) != NULL) { + if (x->rfc3779_addr) { ex_count--; - if (!crit || !v3_addr_is_canonical(addr)) { + if ((loc = X509_get_ext_by_NID(x, NID_sbgp_ipAddrBlock, -1)) < 0 || + !X509_EXTENSION_get_critical(X509_get_ext(x, loc)) || + !v3_addr_is_canonical(x->rfc3779_addr)) { log_validation_status(rc, uri, bad_ipaddrblocks, generation); goto done; } } - if ((asid = X509_get_ext_d2i(x, NID_sbgp_autonomousSysNum, &crit, NULL)) != NULL) { + if (x->rfc3779_asid) { ex_count--; - if (!crit || !v3_asid_is_canonical(asid)) { + if ((loc = X509_get_ext_by_NID(x, NID_sbgp_autonomousSysNum, -1)) < 0 || + !X509_EXTENSION_get_critical(X509_get_ext(x, loc)) || + !v3_asid_is_canonical(x->rfc3779_asid) || + x->rfc3779_asid->rdi != NULL) { log_validation_status(rc, uri, bad_asidentifiers, generation); goto done; } } - if (!addr && !asid) { + if (!x->rfc3779_addr && !x->rfc3779_asid) { log_validation_status(rc, uri, missing_resources, generation); goto done; } @@ -3500,6 +3499,17 @@ static int check_x509(rcynic_ctx_t *rc, goto done; } + if (x->akid) { + ex_count--; + if (!check_aki(rc, uri, w->cert, x->akid, generation)) + goto done; + } + + if (!x->akid && !certinfo->ta) { + log_validation_status(rc, uri, aki_extension_missing, generation); + goto done; + } + if (certinfo->ta) { if (certinfo->crldp.s[0]) { @@ -3509,11 +3519,6 @@ static int check_x509(rcynic_ctx_t *rc, } else { - if (check_aki(rc, uri, w->cert, x->akid, generation)) - ex_count--; - else - goto done; - if (!certinfo->crldp.s[0]) { log_validation_status(rc, uri, crldp_uri_missing, generation); goto done; @@ -3598,6 +3603,130 @@ static int check_x509(rcynic_ctx_t *rc, } /** + * Check a signed CMS object. + */ +static int check_cms(rcynic_ctx_t *rc, + STACK_OF(walk_ctx_t) *wsk, + const uri_t *uri, + path_t *path, + const path_t *prefix, + CMS_ContentInfo **pcms, + X509 **px, + certinfo_t *certinfo, + BIO *bio, + const unsigned char *hash, + const size_t hashlen, + const unsigned char *expected_eContentType, + const size_t expected_eContentType_len, + const int require_inheritance, + const object_generation_t generation) +{ + const ASN1_OBJECT *eContentType = NULL; + STACK_OF(CMS_SignerInfo) *signer_infos = NULL; + STACK_OF(X509) *signers = NULL; + CMS_ContentInfo *cms = NULL; + CMS_SignerInfo *si = NULL; + ASN1_OCTET_STRING *sid = NULL; + X509_NAME *si_issuer = NULL; + ASN1_INTEGER *si_serial = NULL; + hashbuf_t hashbuf; + X509 *x = NULL; + certinfo_t certinfo_; + int i, result = 0; + + assert(rc && wsk && uri && path && prefix && expected_eContentType); + + if (!certinfo) + certinfo = &certinfo_; + + if (!uri_to_filename(rc, uri, path, prefix)) + goto error; + + if (hash) + cms = read_cms(path, &hashbuf); + else + cms = read_cms(path, NULL); + + if (!cms) + goto error; + + if (hash && (hashlen > sizeof(hashbuf.h) || + memcmp(hashbuf.h, hash, hashlen))) { + log_validation_status(rc, uri, digest_mismatch, generation); + if (!rc->allow_digest_mismatch) + goto error; + } + + if (!(eContentType = CMS_get0_eContentType(cms)) || + oid_cmp(eContentType, expected_eContentType, + expected_eContentType_len)) { + log_validation_status(rc, uri, bad_cms_econtenttype, generation); + goto error; + } + + if (CMS_verify(cms, NULL, NULL, NULL, bio, CMS_NO_SIGNER_CERT_VERIFY) <= 0) { + log_validation_status(rc, uri, cms_validation_failure, generation); + goto error; + } + + if (!(signers = CMS_get0_signers(cms)) || sk_X509_num(signers) != 1 || + (x = sk_X509_value(signers, 0)) == NULL) { + log_validation_status(rc, uri, cms_signer_missing, generation); + goto error; + } + + if ((signer_infos = CMS_get0_SignerInfos(cms)) == NULL || + sk_CMS_SignerInfo_num(signer_infos) != 1 || + (si = sk_CMS_SignerInfo_value(signer_infos, 0)) == NULL || + !CMS_SignerInfo_get0_signer_id(si, &sid, &si_issuer, &si_serial) || + sid == NULL || si_issuer != NULL || si_serial != NULL) { + log_validation_status(rc, uri, bad_cms_signer_infos, generation); + goto error; + } + + if (CMS_SignerInfo_cert_cmp(si, x)) { + log_validation_status(rc, uri, cms_ski_mismatch, generation); + goto error; + } + + if (!check_x509(rc, wsk, uri, x, certinfo, generation)) + goto error; + + if (require_inheritance && x->rfc3779_addr) { + for (i = 0; i < sk_IPAddressFamily_num(x->rfc3779_addr); i++) { + IPAddressFamily *f = sk_IPAddressFamily_value(x->rfc3779_addr, i); + if (f->ipAddressChoice->type != IPAddressChoice_inherit) { + log_validation_status(rc, uri, rfc3779_inheritance_required, generation); + goto error; + } + } + } + + if (require_inheritance && x->rfc3779_asid && x->rfc3779_asid->asnum && + x->rfc3779_asid->asnum->type != ASIdentifierChoice_inherit) { + log_validation_status(rc, uri, rfc3779_inheritance_required, generation); + goto error; + } + + if (pcms) { + *pcms = cms; + cms = NULL; + } + + if (px) + *px = x; + + result = 1; + + error: + CMS_ContentInfo_free(cms); + + return result; +} + + + +/** * Load certificate, check against manifest, then run it through all * the check_x509() tests. */ @@ -3707,43 +3836,21 @@ static Manifest *check_manifest_1(rcynic_ctx_t *rc, const object_generation_t generation) { Manifest *manifest = NULL, *result = NULL; - const ASN1_OBJECT *eContentType = NULL; - STACK_OF(X509) *signers = NULL; CMS_ContentInfo *cms = NULL; FileAndHash *fah = NULL; BIO *bio = NULL; - X509 *ee; + X509 *x; int i; assert(rc && wsk && uri && path && prefix); - if (!uri_to_filename(rc, uri, path, prefix) || - (cms = read_cms(path, NULL)) == NULL) - goto done; - - if ((eContentType = CMS_get0_eContentType(cms)) == NULL || - oid_cmp(eContentType, id_ct_rpkiManifest, sizeof(id_ct_rpkiManifest))) { - log_validation_status(rc, uri, bad_cms_econtenttype, generation); - goto done; - } - if ((bio = BIO_new(BIO_s_mem())) == NULL) { logmsg(rc, log_sys_err, "Couldn't allocate BIO for manifest %s", uri->s); goto done; } - if (CMS_verify(cms, NULL, NULL, NULL, bio, CMS_NO_SIGNER_CERT_VERIFY) <= 0) { - log_validation_status(rc, uri, cms_validation_failure, generation); - goto done; - } - - if ((signers = CMS_get0_signers(cms)) == NULL || sk_X509_num(signers) != 1 || - (ee = sk_X509_value(signers, 0)) == NULL) { - log_validation_status(rc, uri, cms_signer_missing, generation); - goto done; - } - - if (!check_x509(rc, wsk, uri, ee, certinfo, generation)) + if (!check_cms(rc, wsk, uri, path, prefix, &cms, &x, certinfo, bio, NULL, 0, + id_ct_rpkiManifest, sizeof(id_ct_rpkiManifest), 1, generation)) goto done; if ((manifest = ASN1_item_d2i_bio(ASN1_ITEM_rptr(Manifest), bio, NULL)) == NULL) { @@ -3793,7 +3900,6 @@ static Manifest *check_manifest_1(rcynic_ctx_t *rc, BIO_free(bio); Manifest_free(manifest); CMS_ContentInfo_free(cms); - sk_X509_free(signers); return result; } @@ -3949,21 +4055,12 @@ static int check_roa_1(rcynic_ctx_t *rc, const size_t hashlen, const object_generation_t generation) { - unsigned char addrbuf[ADDR_RAW_BUF_LEN]; - const ASN1_OBJECT *eContentType = NULL; STACK_OF(IPAddressFamily) *roa_resources = NULL, *ee_resources = NULL; - STACK_OF(CMS_SignerInfo) *signer_infos = NULL; - STACK_OF(X509) *signers = NULL; + unsigned char addrbuf[ADDR_RAW_BUF_LEN]; CMS_ContentInfo *cms = NULL; - CMS_SignerInfo *si = NULL; - ASN1_OCTET_STRING *sid = NULL; - X509_NAME *si_issuer = NULL; - ASN1_INTEGER *si_serial = NULL; - hashbuf_t hashbuf; - ROA *roa = NULL; BIO *bio = NULL; + ROA *roa = NULL; X509 *x = NULL; - certinfo_t certinfo; int i, j, result = 0; unsigned afi, *safi = NULL, safi_, prefixlen; ROAIPAddressFamily *rf; @@ -3971,62 +4068,14 @@ static int check_roa_1(rcynic_ctx_t *rc, assert(rc && wsk && uri && path && prefix); - if (!uri_to_filename(rc, uri, path, prefix)) - goto error; - - if (hash) - cms = read_cms(path, &hashbuf); - else - cms = read_cms(path, NULL); - - if (!cms) - goto error; - - if (hash && (hashlen > sizeof(hashbuf.h) || - memcmp(hashbuf.h, hash, hashlen))) { - log_validation_status(rc, uri, digest_mismatch, generation); - if (!rc->allow_digest_mismatch) - goto error; - } - - if (!(eContentType = CMS_get0_eContentType(cms)) || - oid_cmp(eContentType, id_ct_routeOriginAttestation, - sizeof(id_ct_routeOriginAttestation))) { - log_validation_status(rc, uri, bad_cms_econtenttype, generation); - goto error; - } - if ((bio = BIO_new(BIO_s_mem())) == NULL) { logmsg(rc, log_sys_err, "Couldn't allocate BIO for ROA %s", uri->s); goto error; } - if (CMS_verify(cms, NULL, NULL, NULL, bio, CMS_NO_SIGNER_CERT_VERIFY) <= 0) { - log_validation_status(rc, uri, cms_validation_failure, generation); - goto error; - } - - if (!(signers = CMS_get0_signers(cms)) || sk_X509_num(signers) != 1 || - (x = sk_X509_value(signers, 0)) == NULL) { - log_validation_status(rc, uri, cms_signer_missing, generation); - goto error; - } - - if ((signer_infos = CMS_get0_SignerInfos(cms)) == NULL || - sk_CMS_SignerInfo_num(signer_infos) != 1 || - (si = sk_CMS_SignerInfo_value(signer_infos, 0)) == NULL || - !CMS_SignerInfo_get0_signer_id(si, &sid, &si_issuer, &si_serial) || - sid == NULL || si_issuer != NULL || si_serial != NULL) { - log_validation_status(rc, uri, bad_cms_signer_infos, generation); - goto error; - } - - if (CMS_SignerInfo_cert_cmp(si, x)) { - log_validation_status(rc, uri, cms_ski_mismatch, generation); - goto error; - } - - if (!check_x509(rc, wsk, uri, x, &certinfo, generation)) + if (!check_cms(rc, wsk, uri, path, prefix, &cms, &x, NULL, bio, NULL, 0, + id_ct_routeOriginAttestation, sizeof(id_ct_routeOriginAttestation), + 0, generation)) goto error; if (!(roa = ASN1_item_d2i_bio(ASN1_ITEM_rptr(ROA), bio, NULL))) { @@ -4039,12 +4088,12 @@ static int check_roa_1(rcynic_ctx_t *rc, goto error; } - /* - * ROA issuer doesn't need rights to the ASN, so we don't need to - * check the asID field. - */ + if (ASN1_INTEGER_cmp(roa->asID, asn1_zero) < 0) { + log_validation_status(rc, uri, bad_roa_asID, generation); + goto error; + } - ee_resources = X509_get_ext_d2i(sk_X509_value(signers, 0), NID_sbgp_ipAddrBlock, NULL, NULL); + ee_resources = X509_get_ext_d2i(x, NID_sbgp_ipAddrBlock, NULL, NULL); /* * Extract prefixes from ROA and convert them into a resource set. @@ -4134,7 +4183,6 @@ static int check_roa_1(rcynic_ctx_t *rc, BIO_free(bio); ROA_free(roa); CMS_ContentInfo_free(cms); - sk_X509_free(signers); sk_IPAddressFamily_pop_free(roa_resources, IPAddressFamily_free); sk_IPAddressFamily_pop_free(ee_resources, IPAddressFamily_free); @@ -4199,41 +4247,13 @@ static int check_ghostbuster_1(rcynic_ctx_t *rc, const size_t hashlen, const object_generation_t generation) { - const ASN1_OBJECT *eContentType = NULL; - STACK_OF(X509) *signers = NULL; CMS_ContentInfo *cms = NULL; - hashbuf_t hashbuf; BIO *bio = NULL; - certinfo_t certinfo; + X509 *x; int result = 0; assert(rc && wsk && uri && path && prefix); - if (!uri_to_filename(rc, uri, path, prefix)) - goto error; - - if (hash) - cms = read_cms(path, &hashbuf); - else - cms = read_cms(path, NULL); - - if (!cms) - goto error; - - if (hash && (hashlen > sizeof(hashbuf.h) || - memcmp(hashbuf.h, hash, hashlen))) { - log_validation_status(rc, uri, digest_mismatch, generation); - if (!rc->allow_digest_mismatch) - goto error; - } - - if (!(eContentType = CMS_get0_eContentType(cms)) || - oid_cmp(eContentType, id_ct_rpkiGhostbusters, - sizeof(id_ct_rpkiGhostbusters))) { - log_validation_status(rc, uri, bad_cms_econtenttype, generation); - goto error; - } - #if 0 /* * May want this later if we're going to inspect the VCard. For now, @@ -4245,15 +4265,10 @@ static int check_ghostbuster_1(rcynic_ctx_t *rc, } #endif - if (CMS_verify(cms, NULL, NULL, NULL, bio, CMS_NO_SIGNER_CERT_VERIFY) <= 0) { - log_validation_status(rc, uri, cms_validation_failure, generation); - goto error; - } - - if (!(signers = CMS_get0_signers(cms)) || sk_X509_num(signers) != 1) { - log_validation_status(rc, uri, cms_signer_missing, generation); + if (!check_cms(rc, wsk, uri, path, prefix, &cms, &x, NULL, bio, NULL, 0, + id_ct_rpkiGhostbusters, sizeof(id_ct_rpkiGhostbusters), + 1, generation)) goto error; - } #if 0 /* @@ -4262,15 +4277,11 @@ static int check_ghostbuster_1(rcynic_ctx_t *rc, */ #endif - if (!check_x509(rc, wsk, uri, sk_X509_value(signers, 0), &certinfo, generation)) - goto error; - result = 1; error: BIO_free(bio); CMS_ContentInfo_free(cms); - sk_X509_free(signers); return result; } diff --git a/rcynic/rcynic.py b/rcynic/rcynic.py index 8bdb7786..b7ddb6fc 100644 --- a/rcynic/rcynic.py +++ b/rcynic/rcynic.py @@ -24,13 +24,14 @@ import sys, urlparse, os, getopt from xml.etree.ElementTree import (ElementTree, Element, SubElement, Comment) opt = { - "refresh" : 1800, - "suppress_zero_columns" : True, - "use_colors" : True, - "show_detailed_status" : True, - "show_problems" : False, - "show_summary" : True, - "one_file_per_section" : False } + "refresh" : 1800, + "suppress_zero_columns" : True, + "use_colors" : True, + "show_detailed_status" : True, + "show_problems" : False, + "show_summary" : True, + "suppress_backup_whining" : True, + "one_file_per_section" : False } def usage(msg = 0): f = sys.stderr if msg else sys.stdout @@ -79,13 +80,17 @@ class Label(object): class Validation_Status(object): - def __init__(self, elt, map): + label_map = None + + def __init__(self, elt): self.uri = elt.text.strip() self.timestamp = elt.get("timestamp") self.generation = elt.get("generation") self.hostname = urlparse.urlparse(self.uri).hostname or None self.fn2 = os.path.splitext(self.uri)[1] or None if self.generation else None - self.label = map[elt.get("status")] + self.label = self.label_map[elt.get("status")] + + def stand_up_and_be_counted(self): self.label.sum += 1 @property @@ -96,6 +101,22 @@ class Validation_Status(object): def mood(self): return self.label.mood + @property + def accepted(self): + return self.label.code == "object_accepted" + + @property + def rejected(self): + return self.label.code == "object_rejected" + + @property + def is_current(self): + return self.generation == "current" + + @property + def is_backup(self): + return self.generation == "backup" + html = None body = None @@ -150,9 +171,17 @@ def finish_html(name = None): input = ElementTree(file = sys.stdin if input_file is None else input_file) labels = [Label(elt) for elt in input.find("labels")] -label_map = dict((l.code, l) for l in labels) -validation_status = [Validation_Status(elt, label_map) for elt in input.findall("validation_status")] -del label_map +Validation_Status.label_map = dict((l.code, l) for l in labels) +validation_status = [Validation_Status(elt) for elt in input.findall("validation_status")] + +if opt["suppress_backup_whining"]: + + accepted_current = set(v.uri for v in validation_status if v.is_current and v.accepted) + validation_status = [v for v in validation_status if not v.is_backup or v.uri not in accepted_current] + +for v in validation_status: + v.stand_up_and_be_counted() + if opt["suppress_zero_columns"]: labels = [l for l in labels if l.sum > 0] diff --git a/rpkid/examples/rpki.conf b/rpkid/examples/rpki.conf index 53216b97..84d7109c 100644 --- a/rpkid/examples/rpki.conf +++ b/rpkid/examples/rpki.conf @@ -331,7 +331,7 @@ rpki-root-crl = root.crl # Filename (relative to rootd-base-uri and rpki-root-dir) of the # manifest for rootd's root RPKI certificate -rpki-root-manifest = root.mnf +rpki-root-manifest = root.mft # Up-down protocol class name for RPKI certificate rootd issues to its # one (and only) child @@ -364,7 +364,7 @@ root_cert_sia = rsync://${myrpki::publication_rsync_server}/${myrpki::publicat # root_cert_sia + rpki-root-manifest -root_cert_manifest = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/root.mnf +root_cert_manifest = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/root.mft ################################################################# diff --git a/rpkid/irbe_cli.py b/rpkid/irbe_cli.py index f7593a75..637ad720 100644 --- a/rpkid/irbe_cli.py +++ b/rpkid/irbe_cli.py @@ -227,9 +227,14 @@ class roa_elt(cmd_elt_mixin, rpki.publication.roa_elt): class report_error_elt(reply_elt_mixin, rpki.publication.report_error_elt): pass +class ghostbuster_elt(cmd_elt_mixin, rpki.publication.ghostbuster_elt): + pass + class publication_msg(cmd_msg_mixin, rpki.publication.msg): pdus = dict((x.element_name, x) - for x in (config_elt, client_elt, certificate_elt, crl_elt, manifest_elt, roa_elt, report_error_elt)) + for x in (config_elt, client_elt, certificate_elt, crl_elt, + manifest_elt, roa_elt, report_error_elt, + ghostbuster_elt)) class publication_sax_handler(rpki.publication.sax_handler): pdu = publication_msg diff --git a/rpkid/rpki/__doc__.py b/rpkid/rpki/__doc__.py index 1f9a7ec2..c53de51e 100644 --- a/rpkid/rpki/__doc__.py +++ b/rpkid/rpki/__doc__.py @@ -1382,7 +1382,7 @@ # # @par @c rpki-root-manifest: # Name of file to which rootd should save its -# RPKI manifest. Default is "Root.mnf". +# RPKI manifest. Default is "Root.mft". # # @par @c rpki-subject-pkcs10: # Name of file that rootd should use when saving diff --git a/rpkid/rpki/rootd.py b/rpkid/rpki/rootd.py index 26553b33..44e6af83 100644 --- a/rpkid/rpki/rootd.py +++ b/rpkid/rpki/rootd.py @@ -306,7 +306,7 @@ class main(object): self.rpki_root_cert_file = self.cfg.get("rpki-root-cert") self.rpki_root_cert_uri = self.cfg.get("rpki-root-cert-uri", self.rpki_base_uri + "Root.cer") - self.rpki_root_manifest = self.cfg.get("rpki-root-manifest", "Root.mnf") + self.rpki_root_manifest = self.cfg.get("rpki-root-manifest", "Root.mft") self.rpki_root_crl = self.cfg.get("rpki-root-crl", "Root.crl") self.rpki_subject_cert = self.cfg.get("rpki-subject-cert", "Child.cer") self.rpki_subject_pkcs10 = self.cfg.get("rpki-subject-pkcs10", "Child.pkcs10") diff --git a/rpkid/rpki/rpkid.py b/rpkid/rpki/rpkid.py index 9a9be46e..715a8aa2 100644 --- a/rpkid/rpki/rpkid.py +++ b/rpkid/rpki/rpkid.py @@ -742,7 +742,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): """ Return publication URI for this ca_detail's manifest. """ - return self.ca.sia_uri + self.public_key.gSKI() + ".mnf" + return self.ca.sia_uri + self.public_key.gSKI() + ".mft" def has_expired(self): """ diff --git a/rpkid/rpki/up_down.py b/rpkid/rpki/up_down.py index 009818cb..0eba6b52 100644 --- a/rpkid/rpki/up_down.py +++ b/rpkid/rpki/up_down.py @@ -704,3 +704,5 @@ class cms_msg(rpki.x509.XML_CMS_object): encoding = "UTF-8" schema = rpki.relaxng.up_down saxify = sax_handler.saxify + allow_extra_certs = True + allow_extra_crls = True diff --git a/rpkid/rpki/x509.py b/rpkid/rpki/x509.py index 7bbb47bc..955b8d97 100644 --- a/rpkid/rpki/x509.py +++ b/rpkid/rpki/x509.py @@ -940,11 +940,12 @@ class RSA(DER_object): return self.POW @classmethod - def generate(cls, keylength = 2048): + def generate(cls, keylength = 2048, quiet = False): """ Generate a new keypair. """ - rpki.log.debug("Generating new %d-bit RSA key" % keylength) + if not quiet: + rpki.log.debug("Generating new %d-bit RSA key" % keylength) return cls(POW = rpki.POW.Asymmetric(rpki.POW.RSA_CIPHER, keylength)) def get_public_DER(self): @@ -1052,6 +1053,16 @@ class CMS_object(DER_object): require_crls = False + ## @var allow_extra_certs + # Set this to True to allow CMS messages to contain CA certificates. + + allow_extra_certs = False + + ## @var allow_extra_crls + # Set this to True to allow CMS messages to contain multiple CRLs. + + allow_extra_crls = False + ## @var print_on_der_error # Set this to True to log alleged DER when we have trouble parsing # it, in case it's really a Perl backtrace or something. @@ -1136,36 +1147,41 @@ class CMS_object(DER_object): if self.debug_cms_certs: rpki.log.debug("CMS trusted cert issuer %s subject %s SKI %s" % (x.getIssuer(), x.getSubject(), x.hSKI())) if x.getNotAfter() < now: - raise rpki.exceptions.TrustedCMSCertHasExpired + raise rpki.exceptions.TrustedCMSCertHasExpired("Trusted CMS certificate has expired", "%s (%s)" % (x.getSubject(), x.hSKI())) if not x.is_CA(): - if trusted_ee is not None: - raise rpki.exceptions.MultipleCMSEECert - trusted_ee = x + if trusted_ee is None: + trusted_ee = x + else: + raise rpki.exceptions.MultipleCMSEECert("Multiple CMS EE certificates", *("%s (%s)" % (x.getSubject(), x.hSKI()) for x in ta if not x.is_CA())) store.addTrust(x.get_POW()) if trusted_ee: if self.debug_cms_certs: rpki.log.debug("Trusted CMS EE cert issuer %s subject %s SKI %s" % (trusted_ee.getIssuer(), trusted_ee.getSubject(), trusted_ee.hSKI())) - if certs and (len(certs) > 1 or certs[0].getSubject() != trusted_ee.getSubject() or certs[0].getPublicKey() != trusted_ee.getPublicKey()): - raise rpki.exceptions.UnexpectedCMSCerts # , certs + if len(certs) > 1 or (len(certs) == 1 and + (certs[0].getSubject() != trusted_ee.getSubject() or + certs[0].getPublicKey() != trusted_ee.getPublicKey())): + raise rpki.exceptions.UnexpectedCMSCerts("Unexpected CMS certificates", *("%s (%s)" % (x.getSubject(), x.hSKI()) for x in certs)) if crls: - rpki.log.warn("Ignoring unexpected CMS CRL%s from trusted peer" % ("" if len(crls) == 1 else "s")) + raise rpki.exceptions.UnexpectedCMSCRLs("Unexpected CRLs", *("%s (%s)" % (c.getIssuer(), c.hAKI()) for c in crls)) + else: - if not certs: - raise rpki.exceptions.MissingCMSEEcert # , certs - if len(certs) > 1 or certs[0].is_CA(): - raise rpki.exceptions.UnexpectedCMSCerts # , certs - if not crls: + untrusted_ee = [x for x in certs if not x.is_CA()] + if len(untrusted_ee) < 1: + raise rpki.exceptions.MissingCMSEEcert + if len(untrusted_ee) > 1 or (not self.allow_extra_certs and len(certs) > len(untrusted_ee)): + raise rpki.exceptions.UnexpectedCMSCerts("Unexpected CMS certificates", *("%s (%s)" % (x.getSubject(), x.hSKI()) for x in certs)) + if len(crls) < 1: if self.require_crls: - raise rpki.exceptions.MissingCMSCRL # , crls + raise rpki.exceptions.MissingCMSCRL else: rpki.log.warn("MISSING CMS CRL! Ignoring per self.require_crls setting") - if len(crls) > 1: - raise rpki.exceptions.UnexpectedCMSCRLs # , crls + if len(crls) > 1 and not self.allow_extra_crls: + raise rpki.exceptions.UnexpectedCMSCRLs("Unexpected CRLs", *("%s (%s)" % (c.getIssuer(), c.hAKI()) for c in crls)) for x in certs: if x.getNotAfter() < now: - raise rpki.exceptions.CMSCertHasExpired # , x + raise rpki.exceptions.CMSCertHasExpired("CMS certificate has expired", "%s (%s)" % (x.getSubject(), x.hSKI())) try: content = cms.verify(store) diff --git a/rpkid/tests/Makefile.in b/rpkid/tests/Makefile.in index f86573da..35cd70c3 100644 --- a/rpkid/tests/Makefile.in +++ b/rpkid/tests/Makefile.in @@ -6,7 +6,7 @@ abs_top_builddir = @abs_top_builddir@ all: protocol-samples clean: - rm -rf smoketest.dir left-right-protocol-samples publication-protocol-samples yamltest.dir + rm -rf smoketest.dir left-right-protocol-samples publication-protocol-samples yamltest.dir rcynic.xml rcynic-data protocol-samples: left-right-protocol-samples/.stamp publication-protocol-samples/.stamp diff --git a/rpkid/tests/left-right-protocol-samples.xml b/rpkid/tests/left-right-protocol-samples.xml index 94727558..7b97386d 100644 --- a/rpkid/tests/left-right-protocol-samples.xml +++ b/rpkid/tests/left-right-protocol-samples.xml @@ -973,7 +973,7 @@ fBk4i7H945v/zs7bLLMJxTs8+ao4iCDuknjbGhjWmi9xrTXDtcCXx607rPDkJQcJE2WnRS/U HIA= </list_published_objects> - <list_published_objects self_handle="42" uri="rsync://rpki.example.org/rpki/DEMEtlxZrZes7TNGbe7XwVSMgW0.mnf"> + <list_published_objects self_handle="42" uri="rsync://rpki.example.org/rpki/DEMEtlxZrZes7TNGbe7XwVSMgW0.mft"> MIIHBQYJKoZIhvcNAQcCoIIG9jCCBvICAQMxDTALBglghkgBZQMEAgEwggEfBgsqhkiG9w0B CRABGqCCAQ4EggEKMIIBBgICAWoYDzIwMDkwOTI4MjA1MTQ5WhgPMjAwOTA5MjgyMTUxNDla BglghkgBZQMEAgEwgdIwRBYfREVNRXRseFpyWmVzN1ROR2JlN1h3VlNNZ1cwLmNybAMhAPgd diff --git a/rpkid/tests/publication-protocol-samples.xml b/rpkid/tests/publication-protocol-samples.xml index 12df2785..96b095a7 100644 --- a/rpkid/tests/publication-protocol-samples.xml +++ b/rpkid/tests/publication-protocol-samples.xml @@ -256,7 +256,7 @@ <!-- === --> <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mnf"> + <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"> MIIHCgYJKoZIhvcNAQcCoIIG+zCCBvcCAQMxDTALBglghkgBZQMEAgEwggEeBgsqhkiG9w0B CRABGqCCAQ0EggEJMIIBBQIBEhgPMjAwODA1MjIxODA1MTVaGA8yMDA4MDUyMjE4MDYxNVoG CWCGSAFlAwQCATCB0jBEFh9ZbTVUTzRJYnlDb0pNZ3E2R2o4dG41Mng5U0UuY2VyAyEA4L8Z @@ -295,15 +295,15 @@ </msg> <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mnf"/> + <manifest action="publish" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> </msg> <msg version="1" type="query" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mnf"/> + <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> </msg> <msg version="1" type="reply" xmlns="http://www.hactrn.net/uris/rpki/publication-spec/"> - <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mnf"/> + <manifest action="withdraw" uri="rsync://wombat.invalid/testbed/RIR/R0/1/j7ghjwblCrcCp9ltyPDNzYKPfxc.mft"/> </msg> <!-- === --> diff --git a/rpkid/tests/smoketest.3.yaml b/rpkid/tests/smoketest.3.yaml index 06ab25ea..f7e4d2a9 100644 --- a/rpkid/tests/smoketest.3.yaml +++ b/rpkid/tests/smoketest.3.yaml @@ -51,10 +51,10 @@ kids: ipv4: 10.3.0.0/23 --- #- shell find publication -type f -name '*.roa' -print -exec ../../../utils/print_roa/print_roa {} \; -#- shell find publication -type f -name '*.mnf' -print -exec ../../../utils/print_manifest/print_manifest {} \; +#- shell find publication -type f -name '*.mft' -print -exec ../../../utils/print_manifest/print_manifest {} \; #--- #- shell find publication -type f -name '*.roa' -print -exec ../../../utils/print_roa/print_roa {} \; -#- shell find publication -type f -name '*.mnf' -print -exec ../../../utils/print_manifest/print_manifest {} \; +#- shell find publication -type f -name '*.mft' -print -exec ../../../utils/print_manifest/print_manifest {} \; #--- - shell set -x; rtr_origin=../../../rtr-origin/rtr-origin; $rtr_origin --cronjob rcynic-data/authenticated && $rtr_origin --show --- diff --git a/rpkid/tests/smoketest.py b/rpkid/tests/smoketest.py index 7dfc584c..fa686afd 100644 --- a/rpkid/tests/smoketest.py +++ b/rpkid/tests/smoketest.py @@ -1433,7 +1433,7 @@ rpki-subject-pkcs10 = %(rootd_name)s.subject.pkcs10 rpki-subject-lifetime = %(lifetime)s rpki-root-crl = Bandicoot.crl -rpki-root-manifest = Bandicoot.mnf +rpki-root-manifest = Bandicoot.mft rpki-class-name = Wombat rpki-subject-cert = Wombat.cer @@ -1459,7 +1459,7 @@ authorityKeyIdentifier = keyid:always basicConstraints = critical,CA:true subjectKeyIdentifier = hash keyUsage = critical,keyCertSign,cRLSign -subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:%(rootd_sia)s,1.3.6.1.5.5.7.48.10;URI:%(rootd_sia)sBandicoot.mnf +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:%(rootd_sia)s,1.3.6.1.5.5.7.48.10;URI:%(rootd_sia)sBandicoot.mft sbgp-autonomousSysNum = critical,AS:0-4294967295 sbgp-ipAddrBlock = critical,IPv4:0.0.0.0/0,IPv6:0::/0 certificatePolicies = critical, @rpki_certificate_policy diff --git a/rpkid/tests/testpoke.py b/rpkid/tests/testpoke.py index 26cd29aa..8851a821 100644 --- a/rpkid/tests/testpoke.py +++ b/rpkid/tests/testpoke.py @@ -137,7 +137,7 @@ def do_issue(): q_pdu = rpki.up_down.issue_pdu() req_key = get_PEM("cert-request-key", rpki.x509.RSA, yaml_req) or cms_key sia = ((rpki.oids.name2oid["id-ad-caRepository"], ("uri", yaml_req["sia"][0])), - (rpki.oids.name2oid["id-ad-rpkiManifest"], ("uri", yaml_req["sia"][0] + req_key.gSKI() + ".mnf"))) + (rpki.oids.name2oid["id-ad-rpkiManifest"], ("uri", yaml_req["sia"][0] + req_key.gSKI() + ".mft"))) q_pdu.class_name = yaml_req["class"] q_pdu.pkcs10 = rpki.x509.PKCS10.create_ca(req_key, sia) query_up_down(q_pdu) diff --git a/scripts/analyze-rcynic-history.py b/scripts/analyze-rcynic-history.py index 1713e7ce..7d918198 100644 --- a/scripts/analyze-rcynic-history.py +++ b/scripts/analyze-rcynic-history.py @@ -4,7 +4,7 @@ summaries and run gnuplot to draw some pictures. $Id$ -Copyright (C) 2011 Internet Systems Consortium ("ISC") +Copyright (C) 2011-2012 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -19,77 +19,58 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -show_summary = True -show_sessions = True -show_plot = True -plot_all_hosts = False -plot_to_one = True -plot_to_many = True +plot_all_hosts = False +plot_to_one = True +plot_to_many = True +write_rcynic_xml = True import mailbox, sys, urlparse, os, getopt, datetime, subprocess from xml.etree.cElementTree import (ElementTree as ElementTree, fromstring as ElementTreeFromString) -class Rsync_History(object): +def parse_utc(s): + return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") - timestamp_format = "%Y-%m-%dT%H:%M:%SZ" +class Rsync_History(object): + """ + An Rsync_History object represents one rsync connection. + """ def __init__(self, elt): - self.started = datetime.datetime.strptime(elt.get("started"), self.timestamp_format) - self.finished = datetime.datetime.strptime(elt.get("finished"), self.timestamp_format) self.error = elt.get("error") self.uri = elt.text.strip() self.hostname = urlparse.urlparse(self.uri).hostname or None - self.elapsed = self.finished - self.started - - def __cmp__(self, other): - return (cmp(self.started, other.started) or - cmp(self.finished, other.finished) or - cmp(self.hostname, other.hostname)) + self.elapsed = parse_utc(elt.get("finished")) - parse_utc(elt.get("started")) class Host(object): + """ + A host object represents all the data collected for one host. Note + that it (usually) contains a list of all the sessions in which this + host appears. + """ - def __init__(self, hostname, session_id = None): + def __init__(self, hostname, session_id): self.hostname = hostname - self.session_ids = [] - if session_id is not None: - self.session_ids.append(session_id) + self.session_id = session_id self.elapsed = datetime.timedelta(0) self.connection_count = 0 self.dead_connections = 0 self.uris = set() - self.connections = [] - - def __add__(self, other): - assert self.hostname == other.hostname - result = self.__class__(self.hostname) - for a in ("elapsed", "connection_count", "dead_connections", "session_ids", "connections"): - setattr(result, a, getattr(self, a) + getattr(other, a)) - result.uris = self.uris | other.uris - return result + self.total_connection_time = datetime.timedelta(0) def add_rsync_history(self, h): - self.connection_count += 1 - self.elapsed += h.elapsed - self.dead_connections += int(h.error is not None) - self.connections.append(h) + self.connection_count += 1 + self.elapsed += h.elapsed + self.dead_connections += int(h.error is not None) + self.total_connection_time += h.elapsed def add_uri(self, u): self.uris.add(u) - @property - def session_id(self): - assert len(self.session_ids) == 1 - return self.session_ids[0] - - @property - def session_count(self): - return len(self.session_ids) - - @property - def object_count(self): - return len(self.uris) + def finalize(self): + self.object_count = len(self.uris) + del self.uris @property def failure_rate_percentage(self): @@ -97,28 +78,15 @@ class Host(object): @property def seconds_per_object(self): - return (float((self.elapsed.days * 24 * 3600 + self.elapsed.seconds) * 10**6 + - self.elapsed.microseconds) / - float(self.object_count * self.session_count * 10**6)) + return float(self.elapsed.total_seconds()) / float(self.object_count) @property def objects_per_connection(self): - return (float(self.object_count * self.session_count) / - float(self.connection_count)) - - @property - def scaled_connections(self): - return float(self.connection_count) / float(self.session_count) - - @property - def scaled_elapsed(self): - return self.elapsed / self.session_count + return float(self.object_count) / float(self.connection_count) @property def average_connection_time(self): - return (float(sum(((c.elapsed.days * 24 * 3600 + c.elapsed.seconds) * 10**6 + c.elapsed.microseconds) - for c in self.connections)) / - float(self.connection_count * 10**6)) + return float(self.total_connection_time.total_seconds()) / float(self.connection_count) class Format(object): @@ -135,8 +103,7 @@ class Host(object): except ZeroDivisionError: return self.oops - format = (Format("scaled_elapsed", "Rsync Time", ".10s"), - Format("scaled_connections", "Connections", "d"), + format = (Format("connection_count", "Connections", "d"), Format("object_count", "Objects", "d"), Format("objects_per_connection", "Objects/Connection", ".3f"), Format("seconds_per_object", "Seconds/Object", ".3f"), @@ -157,9 +124,14 @@ class Host(object): return self.format_dict[name](self).strip() class Session(dict): + """ + A session corresponds to one XML file. This is a dictionary of Host + objects, keyed by hostname. + """ - def __init__(self, session_id = None): + def __init__(self, session_id, msg_key): self.session_id = session_id + self.msg_key = msg_key @property def hostnames(self): @@ -168,17 +140,6 @@ class Session(dict): def get_plot_row(self, name, hostnames): return (self.session_id,) + tuple(self[h].format_field(name) if h in self else "" for h in hostnames) - def __add__(self, other): - result = self.__class__() - for h in self.hostnames | other.hostnames: - if h in self and h in other: - result[h] = self[h] + other[h] - elif h in self: - result[h] = self[h] - else: - result[h] = other[h] - return result - def add_rsync_history(self, h): if h.hostname not in self: self[h.hostname] = Host(h.hostname, self.session_id) @@ -189,43 +150,9 @@ class Session(dict): if h and h in self: self[h].add_uri(u) - def dump(self, title, f = sys.stdout): - f.write("\n" + title + "\n" + Host.header + "\n") - for h in sorted(self): - f.write(str(self[h]) + "\n") - -mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) - -sessions = [] - -for msg in mb.itervalues(): - - sys.stderr.write(".") - - assert not msg.is_multipart() - - input = ElementTreeFromString(msg.get_payload()) - - session = Session(input.get("date")) - sessions.append(session) - - for elt in input.findall("rsync_history"): - session.add_rsync_history(Rsync_History(elt)) - - for elt in input.findall("validation_status"): - if elt.get("generation") == "current": - session.add_uri(elt.text.strip()) - -sys.stderr.write("\n") - -summary = sum(sessions, Session()) - -if show_summary: - summary.dump("Summary (%d sessions)" % len(sessions)) - -if show_sessions: - for i, session in enumerate(sessions, 1): - session.dump("Session #%d (%s)" % (i, session.session_id)) + def finalize(self): + for h in self.itervalues(): + h.finalize() def plotter(f, hostnames, field, logscale = False): plotlines = sorted(session.get_plot_row(field, hostnames) for session in sessions) @@ -246,7 +173,7 @@ def plotter(f, hostnames, field, logscale = False): #set format x '%m/%d' set format x '%b%d' #set title '""" + title + """' - plot""" + ",".join(" '-' using 1:2 with lines title '%s'" % h for h in hostnames) + "\n") + plot""" + ",".join(" '-' using 1:2 with linespoints pointinterval 500 title '%s'" % h for h in hostnames) + "\n") for i in xrange(1, n): for plotline in plotlines: f.write("%s %s\n" % (plotline[0], plotline[i].rstrip("%"))) @@ -267,19 +194,65 @@ def plot_one(hostnames, fields): gnuplot.stdin.write("set terminal pdf\n") gnuplot.stdin.write("set output 'analyze-rcynic-history.pdf'\n") for field in fields: - if field not in ("scaled_elapsed", "hostname"): + if field != "hostname": plotter(gnuplot.stdin, hostnames, field, logscale = False) plotter(gnuplot.stdin, hostnames, field, logscale = True) gnuplot.stdin.close() gnuplot.wait() -if show_plot: - if plot_all_hosts: - hostnames = sorted(summary.hostnames) - else: - hostnames = ("rpki.apnic.net", "rpki.ripe.net", "repository.lacnic.net", "rpki.afrinic.net", "arin.rpki.net", "rgnet.rpki.net") - fields = [fmt.attr for fmt in Host.format if fmt.attr not in ("scaled_elapsed", "hostname")] - if plot_to_one: - plot_one(hostnames, fields) - if plot_to_many: - plot_many(hostnames, fields) +mb = mailbox.Maildir("/u/sra/rpki/rcynic-xml", factory = None, create = False) + +sessions = [] + +latest = None + +for i, key in enumerate(mb.iterkeys(), 1): + + sys.stderr.write("\r%s %d/%d..." % ("|\\-/"[i & 3], i, len(mb))) + + assert not mb[key].is_multipart() + + input = ElementTreeFromString(mb[key].get_payload()) + + date = input.get("date") + + sys.stderr.write("%s..." % date) + + session = Session(date, key) + sessions.append(session) + + if latest is None or session.session_id > latest.session_id: + latest = session + + for elt in input.findall("rsync_history"): + session.add_rsync_history(Rsync_History(elt)) + + for elt in input.findall("validation_status"): + if elt.get("generation") == "current": + session.add_uri(elt.text.strip()) + + session.finalize() + +sys.stderr.write("\n") + +if plot_all_hosts: + hostnames = set() + for session in sessions: + hostnames.update(session.hostnames) + hostnames = sorted(hostnames) + +else: + hostnames = ("rpki.apnic.net", "rpki.ripe.net", "repository.lacnic.net", + "rpki.afrinic.net", "arin.rpki.net", "rgnet.rpki.net", + "rpki-pilot.arin.net") + +fields = [fmt.attr for fmt in Host.format if fmt.attr != "hostname"] +if plot_to_one: + plot_one(hostnames, fields) +if plot_to_many: + plot_many(hostnames, fields) + +if write_rcynic_xml and latest is not None: + f = open("rcynic.xml", "wb") + f.write(mb[latest.msg_key].get_payload()) + f.close() |