diff options
| -rw-r--r-- | rpkid/rpki/config.py | 4 | ||||
| -rw-r--r-- | rpkid/rpki/http.py | 2 | ||||
| -rw-r--r-- | rpkid/rpki/left_right.py | 152 | ||||
| -rw-r--r-- | rpkid/rpki/log.py | 25 | ||||
| -rw-r--r-- | rpkid/rpki/publication.py | 17 | ||||
| -rw-r--r-- | rpkid/rpki/rpkid.py | 66 | ||||
| -rw-r--r-- | rpkid/rpki/sql.py | 29 |
7 files changed, 234 insertions, 61 deletions
diff --git a/rpkid/rpki/config.py b/rpkid/rpki/config.py index 6be7558e..a10eae69 100644 --- a/rpkid/rpki/config.py +++ b/rpkid/rpki/config.py @@ -33,7 +33,9 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import ConfigParser, os, re +import ConfigParser +import os +import re ## @var default_filename # Default name of config file if caller doesn't specify one explictly. diff --git a/rpkid/rpki/http.py b/rpkid/rpki/http.py index 6e932d26..7d47826e 100644 --- a/rpkid/rpki/http.py +++ b/rpkid/rpki/http.py @@ -829,7 +829,7 @@ class http_queue(object): log = log_method def __repr__(self): - return rpki.log.log_repr(self, "%s" % addr_to_string(self.hostport)) + return rpki.log.log_repr(self, addr_to_string(self.hostport)) def __init__(self, hostport): self.hostport = hostport diff --git a/rpkid/rpki/left_right.py b/rpkid/rpki/left_right.py index c6089db4..89587f85 100644 --- a/rpkid/rpki/left_right.py +++ b/rpkid/rpki/left_right.py @@ -3,7 +3,7 @@ RPKI "left-right" protocol. $Id$ -Copyright (C) 2009--2011 Internet Systems Consortium ("ISC") +Copyright (C) 2009--2012 Internet Systems Consortium ("ISC") Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -32,9 +32,20 @@ OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ -import rpki.resource_set, rpki.x509, rpki.sql, rpki.exceptions, rpki.xml_utils -import rpki.http, rpki.up_down, rpki.relaxng, rpki.sundial, rpki.log, rpki.roa -import rpki.publication, rpki.async, rpki.rpkid_tasks +import rpki.resource_set +import rpki.x509 +import rpki.sql +import rpki.exceptions +import rpki.xml_utils +import rpki.http +import rpki.up_down +import rpki.relaxng +import rpki.sundial +import rpki.log +import rpki.roa +import rpki.publication +import rpki.async +import rpki.rpkid_tasks ## @var enforce_strict_up_down_xml_sender # Enforce strict checking of XML "sender" field in up-down protocol @@ -60,6 +71,7 @@ class data_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, left_right_name self_handle = None @property + @rpki.sql.cache_reference def self(self): """ Fetch self object to which this object links. @@ -67,6 +79,7 @@ class data_elt(rpki.xml_utils.data_elt, rpki.sql.sql_persistent, left_right_name return self_elt.sql_fetch(self.gctx, self.self_id) @property + @rpki.sql.cache_reference def bsc(self): """ Return BSC object to which this object links. @@ -140,9 +153,16 @@ class self_elt(data_elt): booleans = ("rekey", "reissue", "revoke", "run_now", "publish_world_now", "revoke_forgotten", "clear_replay_protection") - sql_template = rpki.sql.template("self", "self_id", "self_handle", - "use_hsm", "crl_interval", "regen_margin", - ("bpki_cert", rpki.x509.X509), ("bpki_glue", rpki.x509.X509)) + sql_template = rpki.sql.template( + "self", + "self_id", + "self_handle", + "use_hsm", + "crl_interval", + "regen_margin", + ("bpki_cert", rpki.x509.X509), + ("bpki_glue", rpki.x509.X509)) + handles = () use_hsm = False @@ -152,6 +172,9 @@ class self_elt(data_elt): bpki_glue = None cron_tasks = None + def __repr__(self): + return rpki.log.log_repr(self) + @property def bscs(self): """ @@ -373,12 +396,17 @@ class bsc_elt(data_elt): elements = ("signing_cert", "signing_cert_crl", "pkcs10_request") booleans = ("generate_keypair",) - sql_template = rpki.sql.template("bsc", "bsc_id", "bsc_handle", - "self_id", "hash_alg", - ("private_key_id", rpki.x509.RSA), - ("pkcs10_request", rpki.x509.PKCS10), - ("signing_cert", rpki.x509.X509), - ("signing_cert_crl", rpki.x509.CRL)) + sql_template = rpki.sql.template( + "bsc", + "bsc_id", + "bsc_handle", + "self_id", + "hash_alg", + ("private_key_id", rpki.x509.RSA), + ("pkcs10_request", rpki.x509.PKCS10), + ("signing_cert", rpki.x509.X509), + ("signing_cert_crl", rpki.x509.CRL)) + handles = (("self", self_elt),) private_key_id = None @@ -386,6 +414,9 @@ class bsc_elt(data_elt): signing_cert = None signing_cert_crl = None + def __repr__(self): + return rpki.log.log_repr(self, self.bsc_handle) + @property def repositories(self): """ @@ -429,18 +460,27 @@ class repository_elt(data_elt): elements = ("bpki_cert", "bpki_glue") booleans = ("clear_replay_protection",) - sql_template = rpki.sql.template("repository", "repository_id", "repository_handle", - "self_id", "bsc_id", "peer_contact_uri", - ("bpki_cert", rpki.x509.X509), - ("bpki_glue", rpki.x509.X509), - ("last_cms_timestamp", rpki.sundial.datetime)) + sql_template = rpki.sql.template( + "repository", + "repository_id", + "repository_handle", + "self_id", + "bsc_id", + "peer_contact_uri", + ("bpki_cert", rpki.x509.X509), + ("bpki_glue", rpki.x509.X509), + ("last_cms_timestamp", rpki.sundial.datetime)) - handles = (("self", self_elt), ("bsc", bsc_elt)) + handles = (("self", self_elt), + ("bsc", bsc_elt)) bpki_cert = None bpki_glue = None last_cms_timestamp = None + def __repr__(self): + return rpki.log.log_repr(self, self.repository_handle) + @property def parents(self): """ @@ -548,21 +588,34 @@ class parent_elt(data_elt): elements = ("bpki_cms_cert", "bpki_cms_glue") booleans = ("rekey", "reissue", "revoke", "revoke_forgotten", "clear_replay_protection") - sql_template = rpki.sql.template("parent", "parent_id", "parent_handle", - "self_id", "bsc_id", "repository_id", - "peer_contact_uri", "sia_base", - "sender_name", "recipient_name", - ("bpki_cms_cert", rpki.x509.X509), - ("bpki_cms_glue", rpki.x509.X509), - ("last_cms_timestamp", rpki.sundial.datetime)) - - handles = (("self", self_elt), ("bsc", bsc_elt), ("repository", repository_elt)) + sql_template = rpki.sql.template( + "parent", + "parent_id", + "parent_handle", + "self_id", + "bsc_id", + "repository_id", + "peer_contact_uri", + "sia_base", + "sender_name", + "recipient_name", + ("bpki_cms_cert", rpki.x509.X509), + ("bpki_cms_glue", rpki.x509.X509), + ("last_cms_timestamp", rpki.sundial.datetime)) + + handles = (("self", self_elt), + ("bsc", bsc_elt), + ("repository", repository_elt)) bpki_cms_cert = None bpki_cms_glue = None last_cms_timestamp = None + def __repr__(self): + return rpki.log.log_repr(self, self.parent_handle) + @property + @rpki.sql.cache_reference def repository(self): """ Fetch repository object to which this parent object links. @@ -781,18 +834,26 @@ class child_elt(data_elt): elements = ("bpki_cert", "bpki_glue") booleans = ("reissue", "clear_replay_protection") - sql_template = rpki.sql.template("child", "child_id", "child_handle", - "self_id", "bsc_id", - ("bpki_cert", rpki.x509.X509), - ("bpki_glue", rpki.x509.X509), - ("last_cms_timestamp", rpki.sundial.datetime)) + sql_template = rpki.sql.template( + "child", + "child_id", + "child_handle", + "self_id", + "bsc_id", + ("bpki_cert", rpki.x509.X509), + ("bpki_glue", rpki.x509.X509), + ("last_cms_timestamp", rpki.sundial.datetime)) - handles = (("self", self_elt), ("bsc", bsc_elt)) + handles = (("self", self_elt), + ("bsc", bsc_elt)) bpki_cert = None bpki_glue = None last_cms_timestamp = None + def __repr__(self): + return rpki.log.log_repr(self, self.child_handle) + def fetch_child_certs(self, ca_detail = None, ski = None, unique = False): """ Fetch all child_cert objects that link to this child object. @@ -854,7 +915,9 @@ class child_elt(data_elt): raise rpki.exceptions.ClassNameUnknown, "Unknown class name %s" % class_name parent = ca.parent if self.self_id != parent.self_id: - raise rpki.exceptions.ClassNameMismatch, "Class name mismatch: child.self_id = %d, parent.self_id = %d" % (self.self_id, parent.self_id) + raise rpki.exceptions.ClassNameMismatch( + "Class name mismatch: child.self_id = %d, parent.self_id = %d" % ( + self.self_id, parent.self_id)) return ca def serve_destroy_hook(self, cb, eb): @@ -918,6 +981,9 @@ class list_resources_elt(rpki.xml_utils.base_elt, left_right_namespace): attributes = ("self_handle", "tag", "child_handle", "valid_until", "asn", "ipv4", "ipv6") valid_until = None + def __repr__(self): + return rpki.log.log_repr(self, self.self_handle, self.child_handle, self.asn, self.ipv4, self.ipv6) + def startElement(self, stack, name, attrs): """ Handle <list_resources/> element. This requires special handling @@ -965,7 +1031,7 @@ class list_roa_requests_elt(rpki.xml_utils.base_elt, left_right_namespace): self.ipv6 = rpki.resource_set.roa_prefix_set_ipv6(self.ipv6) def __repr__(self): - return rpki.log.log_repr(self, self.asn, self.ipv4, self.ipv6) + return rpki.log.log_repr(self, self.self_handle, self.asn, self.ipv4, self.ipv6) class list_ghostbuster_requests_elt(rpki.xml_utils.text_elt, left_right_namespace): """ @@ -978,6 +1044,8 @@ class list_ghostbuster_requests_elt(rpki.xml_utils.text_elt, left_right_namespac vcard = None + def __repr__(self): + return rpki.log.log_repr(self, self.self_handle, self.parent_handle) class list_published_objects_elt(rpki.xml_utils.text_elt, left_right_namespace): """ @@ -991,6 +1059,9 @@ class list_published_objects_elt(rpki.xml_utils.text_elt, left_right_namespace): obj = None child_handle = None + def __repr__(self): + return rpki.log.log_repr(self, self.self_handle, self.child_handle, self.uri) + def serve_dispatch(self, r_msg, cb, eb): """ Handle a <list_published_objects/> query. The method name is a @@ -1029,6 +1100,9 @@ class list_received_resources_elt(rpki.xml_utils.base_elt, left_right_namespace) attributes = ("self_handle", "tag", "parent_handle", "notBefore", "notAfter", "uri", "sia_uri", "aia_uri", "asn", "ipv4", "ipv6") + def __repr__(self): + return rpki.log.log_repr(self, self.self_handle, self.parent_handle, self.uri, self.notAfter) + def serve_dispatch(self, r_msg, cb, eb): """ Handle a <list_received_resources/> query. The method name is a @@ -1072,6 +1146,9 @@ class report_error_elt(rpki.xml_utils.text_elt, left_right_namespace): error_text = None + def __repr__(self): + return rpki.log.log_repr(self, self.self_handle, self.error_code) + @classmethod def from_exception(cls, e, self_handle = None, tag = None): """ @@ -1114,7 +1191,8 @@ class msg(rpki.xml_utils.msg, left_right_namespace): def fail(e): if not isinstance(e, rpki.exceptions.NotFound): rpki.log.traceback() - r_msg.append(report_error_elt.from_exception(e, self_handle = q_pdu.self_handle, tag = q_pdu.tag)) + r_msg.append(report_error_elt.from_exception( + e, self_handle = q_pdu.self_handle, tag = q_pdu.tag)) cb(r_msg) try: diff --git a/rpkid/rpki/log.py b/rpkid/rpki/log.py index 1f323a36..7c20c3c1 100644 --- a/rpkid/rpki/log.py +++ b/rpkid/rpki/log.py @@ -44,7 +44,6 @@ try: except ImportError: have_setproctitle = False - ## @var enable_trace # Whether call tracing is enabled. @@ -72,6 +71,15 @@ enable_tracebacks = True use_setproctitle = True +## @var proctitle_extra + +# Extra text to include in proctitle display. By default this is the +# tail of the current directory name, as this is often useful, but you +# can set it to something else if you like. If None or the empty +# string, the extra information field will be omitted from the proctitle. + +proctitle_extra = os.path.basename(os.getcwd()) + tag = "" pid = 0 @@ -87,7 +95,10 @@ def init(ident = "rpki", flags = syslog.LOG_PID, facility = syslog.LOG_DAEMON): tag = ident pid = os.getpid() if ident and have_setproctitle and use_setproctitle: - setproctitle.setproctitle("%s (%s)" % (ident, os.path.basename(os.getcwd()))) + if proctitle_extra: + setproctitle.setproctitle("%s (%s)" % (ident, proctitle_extra)) + else: + setproctitle.setproctitle(ident) def set_trace(enable): """ @@ -165,7 +176,15 @@ def log_repr(obj, *tokens): words.append("{%s}" % obj.self.self_handle) except: pass - words.extend(str(token) for token in tokens if token is not None and token != "") + for token in tokens: + if token is not None and token != "": + try: + assert token is not None + words.append(str(token)) + except: + debug("Failed to generate repr() string for object of type %r" % type(token)) + traceback() + words.append("???") if show_python_ids: words.append(" at %#x" % id(obj)) return "<" + " ".join(words) + ">" diff --git a/rpkid/rpki/publication.py b/rpkid/rpki/publication.py index 07905601..69086dfe 100644 --- a/rpkid/rpki/publication.py +++ b/rpkid/rpki/publication.py @@ -73,7 +73,10 @@ class config_elt(control_elt): element_name = "config" elements = ("bpki_crl",) - sql_template = rpki.sql.template("config", "config_id", ("bpki_crl", rpki.x509.CRL)) + sql_template = rpki.sql.template( + "config", + "config_id", + ("bpki_crl", rpki.x509.CRL)) wired_in_config_id = 1 @@ -120,10 +123,14 @@ class client_elt(control_elt): elements = ("bpki_cert", "bpki_glue") booleans = ("clear_replay_protection",) - sql_template = rpki.sql.template("client", "client_id", "client_handle", "base_uri", - ("bpki_cert", rpki.x509.X509), - ("bpki_glue", rpki.x509.X509), - ("last_cms_timestamp", rpki.sundial.datetime)) + sql_template = rpki.sql.template( + "client", + "client_id", + "client_handle", + "base_uri", + ("bpki_cert", rpki.x509.X509), + ("bpki_glue", rpki.x509.X509), + ("last_cms_timestamp", rpki.sundial.datetime)) base_uri = None bpki_cert = None diff --git a/rpkid/rpki/rpkid.py b/rpkid/rpki/rpkid.py index c827386b..15d75b79 100644 --- a/rpkid/rpki/rpkid.py +++ b/rpkid/rpki/rpkid.py @@ -347,7 +347,6 @@ class main(object): except IndexError: self.task_current = None else: - rpki.log.debug("Pulled %r from task queue" % self.task_current) rpki.async.event_defer(self.task_current) def task_run(self): @@ -427,15 +426,22 @@ class ca_obj(rpki.sql.sql_persistent): "ca_id", "last_crl_sn", ("next_crl_update", rpki.sundial.datetime), - "last_issued_sn", "last_manifest_sn", + "last_issued_sn", + "last_manifest_sn", ("next_manifest_update", rpki.sundial.datetime), - "sia_uri", "parent_id", "parent_resource_class") + "sia_uri", + "parent_id", + "parent_resource_class") last_crl_sn = 0 last_issued_sn = 0 last_manifest_sn = 0 + def __repr__(self): + return rpki.log.log_repr(self, repr(self.parent), self.parent_resource_class) + @property + @rpki.sql.cache_reference def parent(self): """ Fetch parent object to which this CA object links. @@ -741,6 +747,9 @@ class ca_detail_obj(rpki.sql.sql_persistent): manifest_published = None latest_ca_cert = None + def __repr__(self): + return rpki.log.log_repr(self, repr(self.ca), self.state, self.ca_cert_uri) + def sql_decode(self, vals): """ Extra assertions for SQL decode of a ca_detail_obj. @@ -750,6 +759,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): assert self.manifest_public_key is None or self.manifest_private_key_id is None or self.manifest_public_key.get_DER() == self.manifest_private_key_id.get_public_DER() @property + @rpki.sql.cache_reference def ca(self): """ Fetch CA object to which this ca_detail links. @@ -1060,6 +1070,7 @@ class ca_detail_obj(rpki.sql.sql_persistent): rpki.log.debug("Created new child_cert %r" % child_cert) else: child_cert.cert = cert + del child_cert.ca_detail child_cert.ca_detail_id = self.ca_detail_id rpki.log.debug("Reusing existing child_cert %r" % child_cert) @@ -1194,6 +1205,9 @@ class child_cert_obj(rpki.sql.sql_persistent): "ski", ("published", rpki.sundial.datetime)) + def __repr__(self): + return rpki.log.log_repr(self, self.uri) + def __init__(self, gctx = None, child_id = None, ca_detail_id = None, cert = None): """ Initialize a child_cert_obj. @@ -1208,19 +1222,28 @@ class child_cert_obj(rpki.sql.sql_persistent): self.sql_mark_dirty() @property + @rpki.sql.cache_reference def child(self): """ Fetch child object to which this child_cert object links. """ return rpki.left_right.child_elt.sql_fetch(self.gctx, self.child_id) - + @property + @rpki.sql.cache_reference def ca_detail(self): """ Fetch ca_detail object to which this child_cert object links. """ return ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) + @ca_detail.deleter + def ca_detail(self): + try: + del self._ca_detail + except AttributeError: + pass + @property def uri_tail(self): """ @@ -1381,6 +1404,9 @@ class revoked_cert_obj(rpki.sql.sql_persistent): ("revoked", rpki.sundial.datetime), ("expires", rpki.sundial.datetime)) + def __repr__(self): + return rpki.log.log_repr(self, repr(self.ca_detail), self.serial, self.revoked) + def __init__(self, gctx = None, serial = None, revoked = None, expires = None, ca_detail_id = None): """ Initialize a revoked_cert_obj. @@ -1395,6 +1421,7 @@ class revoked_cert_obj(rpki.sql.sql_persistent): self.sql_mark_dirty() @property + @rpki.sql.cache_reference def ca_detail(self): """ Fetch ca_detail object to which this revoked_cert_obj links. @@ -1434,6 +1461,7 @@ class roa_obj(rpki.sql.sql_persistent): published = None @property + @rpki.sql.cache_reference def self(self): """ Fetch self object to which this roa_obj links. @@ -1441,12 +1469,20 @@ class roa_obj(rpki.sql.sql_persistent): return rpki.left_right.self_elt.sql_fetch(self.gctx, self.self_id) @property + @rpki.sql.cache_reference def ca_detail(self): """ Fetch ca_detail object to which this roa_obj links. """ return rpki.rpkid.ca_detail_obj.sql_fetch(self.gctx, self.ca_detail_id) + @ca_detail.deleter + def ca_detail(self): + try: + del self._ca_detail + except AttributeError: + pass + def sql_fetch_hook(self): """ Extra SQL fetch actions for roa_obj -- handle prefix lists. @@ -1597,6 +1633,7 @@ class roa_obj(rpki.sql.sql_persistent): resources = rpki.resource_set.resource_bag(v4 = v4, v6 = v6) keypair = rpki.x509.RSA.generate() + del self.ca_detail self.ca_detail_id = ca_detail.ca_detail_id self.cert = ca_detail.issue_ee( ca = ca, @@ -1713,7 +1750,11 @@ class ghostbuster_obj(rpki.sql.sql_persistent): published = None vcard = None + def __repr__(self): + return rpki.log.log_repr(self, self.uri) + @property + @rpki.sql.cache_reference def self(self): """ Fetch self object to which this ghostbuster_obj links. @@ -1721,6 +1762,7 @@ class ghostbuster_obj(rpki.sql.sql_persistent): return rpki.left_right.self_elt.sql_fetch(self.gctx, self.self_id) @property + @rpki.sql.cache_reference def ca_detail(self): """ Fetch ca_detail object to which this ghostbuster_obj links. @@ -1906,17 +1948,13 @@ class publication_queue(object): return self._add( uri, obj, repository, handler, cls.make_withdraw) def call_pubd(self, cb, eb): - if self.empty(): + def loop(iterator, rid): + rpki.log.debug("Calling pubd[%r]" % self.repositories[rid]) + self.repositories[rid].call_pubd(iterator, eb, self.msgs[rid], self.handlers) + def done(): + self.clear() cb() - else: - def loop(iterator, rid): - rpki.log.debug("Calling pubd[%r]" % self.repositories[rid]) - self.repositories[rid].call_pubd(iterator, eb, self.msgs[rid], self.handlers) - def done(): - rpki.log.debug("Publication complete") - self.clear() - cb() - rpki.async.iterator(self.repositories, loop, done) + rpki.async.iterator(self.repositories, loop, done) @property def size(self): diff --git a/rpkid/rpki/sql.py b/rpkid/rpki/sql.py index 553bea83..1bb607cc 100644 --- a/rpkid/rpki/sql.py +++ b/rpkid/rpki/sql.py @@ -374,3 +374,32 @@ class sql_persistent(object): """ pass + +def cache_reference(func): + """ + Decorator for use with property methods which just do an SQL lookup based on an ID. + Check for an existing reference to the object, just return that if we find it, + otherwise perform the SQL lookup. + + Not 100% certain this is a good idea, but I //think// it should work well with the + current weak reference SQL cache, so long as we create no circular references. + So don't do that. + """ + + attr_name = "_" + func.__name__ + + def wrapped(self): + try: + value = getattr(self, attr_name) + assert value is not None + except AttributeError: + value = func(self) + if value is not None: + setattr(self, attr_name, value) + return value + + wrapped.__name__ = func.__name__ + wrapped.__doc__ = func.__doc__ + wrapped.__dict__.update(func.__dict__) + + return wrapped |