diff options
Diffstat (limited to 'myrpki')
30 files changed, 5205 insertions, 0 deletions
diff --git a/myrpki/Makefile b/myrpki/Makefile new file mode 100644 index 00000000..28534c63 --- /dev/null +++ b/myrpki/Makefile @@ -0,0 +1,36 @@ +# $Id$ + +all: myrpki.rng + +relaxng: myrpki.rng + xmllint --noout --relaxng myrpki.rng `find test -type f -name '*.xml'` + +lint: myrpki.xml myrpki.rng + xmllint --noout --relaxng myrpki.rng myrpki.xml + +myrpki.rng: myrpki.rnc + trang myrpki.rnc myrpki.rng + +parse: myrpki.xml all + python xml-parse-test.py + +clean: + rm -rf *.xml bpki test screenlog.* .OpenSSL.whines.unless.I.set.this + python sql-cleaner.py + +format: myrpki.xml + xmllint --format myrpki.xml + +graph: + find . -type d -path '*/bpki/*' | while read b; do python ../scripts/x509-dot.py $$b | unflatten -l 8 -f | dot -T ps2 | ps2pdf - $$b/graph.pdf; done + +verify: + sh verify-bpki.sh + +backup: + python sql-dumper.py + tar cvvzf test.$$(TZ='' date +%Y.%m.%d.%H.%M.%S).tgz screenlog.* test backup.*.sql + rm backup.*.sql + +test: myrpki.rng + MYRPKI_RNG=`pwd`/myrpki.rng python yamltest.py diff --git a/myrpki/POW b/myrpki/POW new file mode 120000 index 00000000..43fccd7b --- /dev/null +++ b/myrpki/POW @@ -0,0 +1 @@ +../pow/buildlib/POW
\ No newline at end of file diff --git a/myrpki/README b/myrpki/README new file mode 100644 index 00000000..ac80a3d3 --- /dev/null +++ b/myrpki/README @@ -0,0 +1,484 @@ +$Id$ + +INTRODUCTION + +The design of rpkid and friends assumes that certain tasks can be +thrown over the wall to the registry's back end operation. This was a +deliberate design decision to allow rpkid et al to remain independent +of existing database schema, business PKIs, and so forth that a +registry might already have. All very nice, but it leaves someone who +just wants to test the tools or who has no existing back end with a +fairly large programming project. The tools in this directory attempt +to fill that gap. + +This is a basic implementation of what a registry back end would need +to use rpkid and friends. These tools do not use every available +option, nor are they necessarily as efficient as possible. Large +registries will almost certainly want to roll their own tools, perhaps +using these as a starting point. Nevertheless, we hope that these +tools will at least provide a useful example. + +The primary tool here is a single command line Python program: +myrpki.py. myrpki has a number of commands, most of which are used +for initial setup, some of which are used on an ongoing basis. myrpki +can be run either in an interactive mode or by passing a single +command on the command line when starting the program; the former mode +is intended to be somewhat human-friendly, the latter mode is useful +in scripting, cron jobs, and automated testing. + +myrpki use has two distinct phases: setup and data maintenance. The +setup phase is primarily about constructing the "business PKI" (BPKI) +certificates that the daemons use to authenticate CMS and HTTPS +messages and obtaining the service URLs needed to configure the +daemons. The data maintenance phase is about configuring local data +into the daemons. + +myrpki uses the OpenSSL command line tool for almost all operations on +keys and certificates; the one exception to this is the comamnd which +talks directly to the daemons, as this command uses the same +communication libraries as the daemons themselves do. The intent +behind using the OpenSSL command line tool for everything else is to +allow all the other commands to be run without requiring all the +auxiliary packages upon which the daemons depend; this can be useful, +eg, if one wants to run the back-end on a laptop while running the +daemons on a server, in which case one might prefer not to have to +install a bunch of unnecessary packages on the laptop. + +During setup phase myrpki generates and processes small XML messages +which it expects the user to ship to and from its parents, children, +etc via some out-of-band means (email, perhaps with PGP signatures, +USB stick, we really don't care). During data maintenance phase, +myrpki does something similar with another XML file, to allow hosting +of RPKI services; in the degenerate case where an entity is just +self-hosting (ie, is running the daemons for itself, and only for +itself), this latter XML file need not be sent anywhere. + +The basic idea here is that a user who has resources maintains a set +of .csv files containing a text representation of the data needed by +the back-end, along with a configuration file containing other +parameters. The intent is that these be very simple files that are +easy to generate either by hand or as a dump from relational database, +spreadsheet, awk script, whatever works in your environment. Given +these files, the user then runs myrpki to extract the relevant +information and encode everything about its back end state into an XML +file, which can then be shipped to the appropriate other party. + +Many of the myrpki commands which process XML input write out a new +XML file, either in place or as an entirely new file; in general, +these files need to be sent back to the party that sent the original +file. Think of all this as a very slow packet-based communication +channel, where each XML file is a single packet. In setup phase, +there's generally a single round-trip per setup conversation; in the +data maintenance phase, the same XML file keeps bouncing back and +forth between hosted entity and hosting entity. + +Note that, as certificates and CRLs have expiration and nextUpdate +values, a low-level cycle of updates passing between resource holder +and rpkid operator will be necessary as a part of steady state +operation. [The current version of these tools does not yet +regenerate these expiring objects, but fixing this will be a +relatively minor matter.] + +The third important kind of file in this system is the configuration +file for myrpki. This contains a number of sections, some of which +are for myrpki, others of which are for the OpenSSL command line tool, +still others of which are for the various RPKI daemon programs. The +examples/ subdirectory contains a commented version of the +configuration file that explains the various parameters. + +The .csv files read by myrpki are (now) misnamed: formerly, they were +the "excel-tab" format from the Python csv library, but early users +kept trying to make the colums line up, which didn't do what the users +expected. So now these files are just whitespace-delimted, as a +program like "awk" would understand. + +Keep reading, and don't panic. + +The default configuration file name is myrpki.conf. You can change +this using the "-c" option when invoking myrpki, or by setting the +environment variable MYRPKI_CONF. + +See examples/myrpki.conf for details on the variables that you can +(and in some cases must) set. + +See examples/*.csv for commented examples of the several CSV files. +Note that the comments themselves are not legal CSV, they're just +present to make it easier to understand the examples. + + +GETTING STARTED -- OVERVIEW + +Which process you need to follow depends on whether you are running +rpkid yourself or will be hosted by somebody else. We call the first +case "self-hosted", because the software treats running rpkid to +handle resources that you yourself hold as if you are an rpkid +operator who is hosting an entity that happens to be yourself. + +"$top" in the following refers to wherever you put the +subvert-rpki.hactrn.net code. Once we have autoconf and "make +install" targets, this will be some system directory or another; for +now, it's wherever you checked out a copy of the code from the +subversion repository or unpacked a tarball of the code. + +Most of the setup process looks the same for any resource holder, +regardless of whether they are self-hosting or not. The differences +come in the data maintenence phase. + +The steps needed during setup phase are: + +0) Write a configuration file (copy $top/myrpki/examples/myrpki.conf + and edit as needed). You need to configure the [myrpki] section; + in theory, the rest of the file should be ok as it is, at least for + simple use. You also need to create (either by hand or by dumping + from a database, spreadsheet, whatever) the CSV files describing + prefixes and ASNs you want to allocate to your children and ROAs + you want created. + +1) Initialization ("initialize" command). This creates the local BPKI + and other data structures that can be constructed just based on + local data such as the config file. Other than some internal data + structures, the main output of this step is the "identity.xml" file, + which is used as input to later stages. + + In theory it should be safe to run the "initialize" command more + than once, in practice this has not (yet) been tested. + +2) Send (email, USB stick, carrier pigeon) identity.xml to each of your + parents. This tells each of your parents what you call yourself, + and supplies each parent with a trust anchor for your + resource-holding BPKI. + +3) Each of your parents runs the "configure_child" command, giving the + identity.xml you supplied as input. This registers your data with + the parent, including BPKI cross-registration, and generates a + return message containing your parent's BPKI trust anchors, a + service URL for contacting your parent via the "up-down" protocol, + and (usually) either an offer of publication service (if your parent + operates a repository) or a referral from your parent to whatever + publication service your parent does use. Referrals include a + CMS-signed authorization token that the repository operator can use + to determine that your parent has given you permission to home + underneath your parent in the publication tree. + +4) Each of your parents sends (...) back the response XML file + generated by the "configure_child" command. + +5) You feed the response message you just got into myrpki using the + "configure_parent" command. This registers the parent's information + in your database, including BPKI cross-certification, and processes + the repository offer or referral to generate a publication request + message. + +6) You send (...) the publication request message to the repository. + The <contact_info/> element in the request message should (in + theory) provide some clue as to where you should send this. + +7) The repository operator processes your request using myrpki's + "configure_publication_client" command. This registers your + information, including BPKI cross-certification, and generates a + response message containing the repository's BPKI trust anchor and + service URL. + +8) Repository operator sends (...) the publication confirmation message + back to you. + +9) You process the publication confirmation message using myrpki's + "configure_repository" command. + +At this point you should, in theory, have established relationships, +exchanged trust anchors, and obtained service URLs from all of your +parents and repositories. The last setup step is establishing a +relationship with your RPKI service host, if you're not self-hosted, +but as this is really just the first message of an ongoing exchange +with your host, it's handled by the data maintenance commands. + +The two commands used in data maintenence phase are +"configure_resources" and "configure_daemons". The first is used by +the resource holder, the second is used by the host. In the +self-hosted case, it is not necessary to run "configure_resources" at +all, myrpki will run it for you automatically. + +GETTING STARTED -- CONFIGURATION FILE + +The current sample configuration file should, in theory, be much +simpler to use than in earlier versions of this code. The sample +configuration uses a simple macro-expansion mechanism to place all of +the configuration data you need to touch into the [myrpki] section; +the rest of the configuration file is for the various daemons and +other tools, and is entirely configured via references to the values +defined in the [myrpki] section. + +GETTING STARTED -- HOSTED CASE + +The basic steps involved in getting started for a resource holder who +is being hosted by somebody else are: + +a) Run through steps (0)-(9), above. + +b) Run the configure_resources command to generate myrpki.xml. + +c) Send myrpki.xml to the rpkid operator who will be hosting you. + +d) Wait for your rpkid operator to ship you back an updated XML file + containing a PKCS #10 certificate request for the BPKI signing + context (BSC) created by rpkid. + +e) Run configure_resources again with the XML file received in step + (d), to issue the BSC certificate and update the XML file again to + contain the newly issued BSC certificate. + +f) Send the updated XML file back to your rpkid operator. + +At this point you're done with initial setup. You will need to run +configure_resources again whenever you make any changes to your +configuration file or CSV files. [Once myrpki knows how to update +BPKI CRLs, you will also need to run configure_resources periodically +to keep your BPKI CRLs up to date.] Any time you run +configure_resources myrpki, you should send the updated XML file to +your rpkid operator, who will [generally?] send you a further updated +XML file in response. + +GETTING STARTED -- SELF-HOSTED CASE + +The first few steps involved in getting started for a self-hosted +resource holder (that is, a resource holder that runs its own copy of +rpkid) are the same as in the hosted case above; after that the +process diverges. + +The [current] steps are: + +a) See rpkid/doc/Installation, and follow the basic installation + instructions there to build the RFC-3779-aware OpenSSL code and + associated Python extension module. + +b) Run through steps (0)-(9), above. + +c) Next, you need to set up the MySQL databases that rpkid et al will + use. The MySQL database, username, and password values all need to + match the ones you specified in myrpki.conf. There are two + different ways you can do this: + + i) You can use the sql-setup.py script, which prompts you for your + MySQL root password then attempts to do everything else + automatically using values from myrpki.conf; or + + ii) You can do it manually. + + The first approach is simple: + + $ python sql-setup.py + Please enter your MySQL root password: + + The script should tell you what databases it creates. You can use + the -v option if you want to see more details about what it's doing. + + If you'd prefer to do the SQL setup manually, perhaps because you + have valuable data in other MySQL databases and you don't want to + trust some random setup script with your MySQL root password, + you'll need to use the MySQL command line tool, as follows: + + $ mysql -u root -p + + mysql> CREATE DATABASE irdb_database; + mysql> GRANT all ON irdb_database.* TO irdb_user@localhost IDENTIFIED BY 'irdb_password'; + mysql> USE irdb_database; + mysql> SOURCE $top/rpkid/irdbd.sql; + mysql> CREATE DATABASE rpki_database; + mysql> GRANT all ON rpki_database.* TO rpki_user@localhost IDENTIFIED BY 'rpki_password'; + mysql> USE rpki_database; + mysql> SOURCE $top/rpkid/rpkid.sql; + mysql> COMMIT; + mysql> quit + + where "irdb_database", "irdb_user", "irdb_password", + "rpki_database", "rpki_user", and "rpki_password" are the + appropriate values from your configuration file. + + If you are running pubd and doing manual SQL setup, you'll also + have to do: + + $ mysql -u root -p + mysql> CREATE DATABASE pubd_database; + mysql> GRANT all ON pubd_database.* TO pubd_user@localhost IDENTIFIED BY 'pubd_password'; + mysql> USE pubd_database; + mysql> SOURCE $top/rpkid/pubd.sql; + mysql> COMMIT; + mysql> quit + +d) If you are running your own publication repository (that is, if you + are running pubd), you will also need to set up an rsyncd server or + configure your existing one to serve pubd's output. There's a + sample configuration file in $top/myrpki/examples/rsyncd.conf, but + you may need to do something more complicated if you are already + running rsyncd for other purposes. See the rsync(1) and + rsyncd.conf(5) manual pages for more details. + +e) Start the daemons. You can use $top/myrpki/start-servers.py to do + this, or write your own script. + + If you intend to run pubd, you should make sure that the directory + you specified as publication_base_directory exists and + is writable by the userid that will be running pubd, and should + also make sure to start rsyncd. + +f) Run myrpki's configure_daemons command, twice, with no arguments. + + You need to run the command twice because myrpki has to ask rpkid + to create a keypair and generate a certification request for the + BSC. The first pass does this, the second processes the + certification request, issues the BSC, and loads the result into + rpkid. [Yes, we could automate this somehow, if necessary.] + +At this point, if everything went well, rpkid should be up, +configured, and starting to obtain resource certificates from its +parents, generate CRLs and manifests, and so forth. At this point you +should go figure out how to use the relying party tool, rcynic: see +$top/rcynic/README if you haven't already done so. + +If and when you change your CSV files, you should run +configure_daemons again to feed the changes into the daemons. + +GETTING STARTED -- HOSTING CASE + +If you are running rpkid not just for your own resources but also to +host other resource holders (see "HOSTED CASE" above), your setup will +be almost the same as in the self-hosted case (see "SELF-HOSTED CASE", +above), with one procedural change: you will need to tell +configure_daemons to process the XML files produced by the resource +holders you are hosting. You do this by specifying the names of all +those XML files on as arguments to the configure_daemons command. So, +if you are hosting two friends, Alice and Bob, then, everywhere the +instructions for the self-hosted case say to run configure_daemons +with no arguments, you will instead run it with the names of Alice's +and Bob's XML files as arguments. + +Note that configure_daemons sometimes modifies these XML files, in +which case it will write them back to the same filenames. While it is +possible to figure out the set of circumstances in which this will +happen (at present, only when myrpki has to ask rpkid to create a new +BSC keypair and PKCS #10 certificate request), it may be easiest just +to ship back an updated copy of the XML file after every you run +configure_daemons. + +GETTING STARTED -- "PURE" HOSTING CASE + +In general we assume that anybody who bothers to run rpkid is also a +resource holder, but the software does not insist on this. + +[Er, well, rpkid doesn't, but myrpki now does -- "pure" hosting was an +unused feature that fell by the wayside while simplifying the user +interface. It would be relatively straightforward to add it back if +we ever need it for anything, but the mechanism it used to use no +longer exists -- the old [myirbe] section of the config file has been +collapsed into the [myrpki] section, so testing for existance of the +[myrpki] section no longer works. So we'll need an explicit +configuration option, no big deal, just not worth chasing now.] + +A (perhaps) plausible use for this capability would be if you are an +rpkid-running resource holder who wants for some reason to keep the +resource-holding side of your operation completely separate from the +rpkid-running side of your operation. This is essentially the +pure-hosting model, just with an internal hosted entity within a +different part of your own organization. + +UPGRADING FROM OLD MYRPKI TOOLS + +There's a script that attempts to upgrade from the previous version of +the myrpki tools (myirbe scripts, parents.csv file, etcetera). The +conversion script is not well tested, so taking a backup (including an +SQL dump) FIRST is STRONGLY recommended. The script attempts to read +all the necessary settings out of your old myrpki.conf file and the +obsolete {parents,children,pubclients}.csv files, and writes out a new +configuration file (myrpki.conf.new) and a set of "entitydb" files +(the local XML database used by the current myrpki program). To use +the conversion script, just run + +$ python convert-from-csv-to-entitydb.py + +with no arguments in the directory where your old myrpki.conf and .csv +files reside. See the script itself for available command line +options, most of which override various filenames. + +Note that the conversion script will not rename existing BPKI +directories to the new convention (./bpki/{resources,servers}/), +instead it will write out myrpki.conf.new using the old directory +names (./bpki.{myrpki,myirbe}/); if you want to switch to the new +convention, move the directories yourself and edit the .conf file to +match. The script does not delete any of the old files, so you'll +want to clean up yourself after you're sure the conversion worked. + +Be warned that the old file format contains less information than the +new XML files do, so in some cases the conversion script is just +making stuff up as best it can. In theory, the cases where it has to +do this will not matter, but this has not been tested yet. + +TROUBLESHOOTING + +If you run into trouble setting up this package, the first thing to do +is categorize the kind of trouble you are having. If you've gotten +far enough to be running the daemons, check their log files. If +you're seeing Python exceptions, read the error messages. If you're +getting TLS errors, check to make sure that you're using all the right +BPKI certificates and service contact URLs. + +TLS configuration errors are, unfortunately, notoriously difficult to +debug, because connection failures due to misconfiguration happen +early, deep in the guts of the OpenSSL TLS code, where there isn't +enough application context available to provide useful error messages. + +If you've completed the steps above, everything appears to have gone +OK, but nothing seems to be happening, the first thing to do is check +the logs to confirm that nothing is actively broken. rpkid's log +should include messages telling you when it starts and finishes its +internal "cron" cycle. It can take several cron cycles for resources +to work their way down from your parent into a full set of +certificates and ROAs, so have a little patience. rpkid's log should +also include messages showing every time it contacts its parent(s) or +attempts to publish anything. + +rcynic in fully verbose mode provides a fairly detailed explanation of +what it's doing and why objects that fail have failed. + +You can use rsync (sic) to examine the contents of a publication +repository one directory at a time, without attempting validation, by +running rsync with just the URI of the directory on its command line: + + $ rsync rsync://rpki.example.org/where/ever/ + +[Maybe there should be something here explaining how to use +irbe_cli.py for debugging, but the syntax is fairly obscure as it's +just a command line interface to the left-right and publication +protocols -- almost certainly want a friendlier tool for +troubleshooting.] + +KNOWN ISSUES + +The lxml package provides a Python interface to the Gnome libxml2 and +libxslt C libraries. This code has been quite stable for several +years, but initial testing with lxml compiled and linked against a +newer version of libxml2 ran into problems (specifically, gratuitous +RelaxNG schema validation failures). libxml2 2.7.3 worked; libxml2 +2.7.5 did not work on the test machine in question. Reverting to +libxml2 2.7.3 fixed the problem. Rewriting the two lines of Python +code that were triggering the lxml bug appears to have solved the +problem, so the code now works properly with libxml 2.7.5, but if you +start seeing weird XML validation failures, it might be another +variation of this lxml bug. + +An earlier version of this code ran into problems with what appears to +be an implementation restriction in the the GNU linker ("ld") on +64-bit hardware, resulting in obscure build failures. The workaround +for this required use of shared libraries and is somewhat less +portable than the original code, but without it the code simply would +not build in 64-bit environments with the GNU tools. The current +workaround appears to behave properly, but the workaround requires +that the pathname to the RFC-3779-aware OpenSSL shared libraries be +built into the _POW.so Python extension module. At the moment, in the +absence of "make install" targets for the Python code and libraries, +this means the build directory; eventually, once we're using autoconf +and installation targets, this will be the installation directory. If +necessary, you can override this by setting the LD_LIBRARY_PATH +environment variable, see the ld.so man page for details. This is a +relatively minor variation on the usual build issues for shared +libraries, it's just annoying because shared libraries should not be +needed here and would not be if not for this GNU linker issue. diff --git a/myrpki/apnic-to-csv.py b/myrpki/apnic-to-csv.py new file mode 100644 index 00000000..54e9137c --- /dev/null +++ b/myrpki/apnic-to-csv.py @@ -0,0 +1,49 @@ +""" +Parse APNIC "Extended Allocation and Assignment" reports and write +out (just) the RPKI-relevant fields in myrpki-format CSV syntax. + +$Id$ + +Copyright (C) 2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import csv, myrpki, rpki.ipaddrs + +translations = dict((src, dst) for src, dst in myrpki.csv_reader("translations.csv", columns = 2)) + +asns = myrpki.csv_writer("asns.csv") +prefixes = myrpki.csv_writer("prefixes.csv") + +for line in open("delegated-apnic-extended-latest"): + + line = line.rstrip() + + if not line.startswith("apnic|") or line.endswith("|summary"): + continue + + registry, cc, rectype, start, value, date, status, opaque_id = line.split("|") + + assert registry == "apnic" + + opaque_id = translations.get(opaque_id, opaque_id) + + if rectype == "asn": + asns.writerow((opaque_id, "%s-%s" % (start, int(start) + int(value) - 1))) + + elif rectype == "ipv4": + prefixes.writerow((opaque_id, "%s-%s" % (start, rpki.ipaddrs.v4addr(rpki.ipaddrs.v4addr(start) + long(value) - 1)))) + + elif rectype == "ipv6": + prefixes.writerow((opaque_id, "%s/%s" % (start, value))) diff --git a/myrpki/arin-to-csv.py b/myrpki/arin-to-csv.py new file mode 100644 index 00000000..55e5762a --- /dev/null +++ b/myrpki/arin-to-csv.py @@ -0,0 +1,121 @@ +""" +Parse a WHOIS research dump and write out (just) the RPKI-relevant +fields in myrpki-format CSV syntax. + +NB: The input data for this script comes from ARIN under an agreement +that allows research use but forbids redistribution, so if you think +you need a copy of the data, please talk to ARIN about it, not us. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import gzip, csv, myrpki + +class Handle(object): + + want_tags = () + + debug = False + + def set(self, tag, val): + if tag in self.want_tags: + setattr(self, tag, "".join(val.split(" "))) + + def check(self): + for tag in self.want_tags: + if not hasattr(self, tag): + return False + if self.debug: + print repr(self) + return True + +class ASHandle(Handle): + + want_tags = ("ASHandle", "ASNumber", "OrgID") + + def __repr__(self): + return "<%s %s.%s %s>" % (self.__class__.__name__, + self.OrgID, self.ASHandle, self.ASNumber) + + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((ctx.translations.get(self.OrgID, self.OrgID), self.ASNumber)) + +class NetHandle(Handle): + + NetType = None + + want_tags = ("NetHandle", "NetRange", "NetType", "OrgID") + + def finish(self, ctx): + if self.NetType in ("allocation", "assignment") and self.check(): + ctx.prefixes.writerow((ctx.translations.get(self.OrgID, self.OrgID), self.NetRange)) + + def __repr__(self): + return "<%s %s.%s %s %s>" % (self.__class__.__name__, + self.OrgID, self.NetHandle, + self.NetType, self.NetRange) + +class V6NetHandle(NetHandle): + + want_tags = ("V6NetHandle", "NetRange", "NetType", "OrgID") + + def __repr__(self): + return "<%s %s.%s %s %s>" % (self.__class__.__name__, + ctx.translations.get(self.OrgID, self.OrgID), + self.V6NetHandle, self.NetType, self.NetRange) + +class main(object): + + types = { + "ASHandle" : ASHandle, + "NetHandle" : NetHandle, + "V6NetHandle" : V6NetHandle } + + translations = {} + + @staticmethod + def parseline(line): + tag, sep, val = line.partition(":") + assert sep, "Couldn't find separator in %r" % line + return tag.strip(), val.strip() + + def __init__(self): + self.asns = myrpki.csv_writer("asns.csv") + self.prefixes = myrpki.csv_writer("prefixes.csv") + try: + self.translations = dict((src, dst) for src, dst in myrpki.csv_reader("translations.csv", columns = 2)) + except IOError: + pass + f = gzip.open("arin_db.txt.gz") + cur = None + for line in f: + line = line.expandtabs().strip() + if not line: + if cur: + cur.finish(self) + cur = None + elif not line.startswith("#"): + tag, val = self.parseline(line) + if cur is None: + cur = self.types[tag]() if tag in self.types else False + if cur: + cur.set(tag, val) + if cur: + cur.finish(self) + +main() diff --git a/myrpki/convert-from-csv-to-entitydb.py b/myrpki/convert-from-csv-to-entitydb.py new file mode 100644 index 00000000..282d2e75 --- /dev/null +++ b/myrpki/convert-from-csv-to-entitydb.py @@ -0,0 +1,233 @@ +""" +Convert {parents,children,pubclients}.csv into new XML formats. + +$Id$ + +Copyright (C) 2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, csv, re, os, getopt, sys, base64, urlparse +import rpki.sundial, myrpki, rpki.config + +from lxml.etree import Element, SubElement, ElementTree + +section_regexp = re.compile("\s*\[\s*(.+?)\s*\]\s*$") +variable_regexp = re.compile("\s*([-a-zA-Z0-9_]+)(\s*=\s*)(.+?)\s*$") + +cfg_file = "myrpki.conf" +template_file = os.path.join(os.path.dirname(sys.argv[0]), "examples", "myrpki.conf") +new_cfg_file = None +preserve_valid_until = False + +opts, argv = getopt.getopt(sys.argv[1:], "c:hn:pt:?", ["config=", "new_config=", "preserve_valid_until", "template_config=", "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-n", "--new_config"): + new_cfg_file = a + elif o in ("-p", "--preserve_valid_until"): + preserve_valid_until = True + elif o in ("-t", "--template_config"): + template_file = a +if argv: + raise RuntimeError, "Unexpected arguments %r" % (argv,) +if os.path.samefile(cfg_file, template_file): + raise RuntimeError, "Old config and template for new config can't be the same file" +if new_cfg_file is None: + new_cfg_file = cfg_file + ".new" +if os.path.exists(new_cfg_file): + raise RuntimeError, "%s already exists, NOT overwriting" % new_cfg_file + +cfg = rpki.config.parser(cfg_file) + +# These have no counterparts in new config file, just read them from old + +repository_bpki_certificate = cfg.get(option = "repository_bpki_certificate", section = "myrpki") +repository_handle = cfg.get(option = "repository_handle", section = "myrpki") +parents_csv = cfg.get(option = "parents_csv", section = "myrpki", default = "parents.csv") +children_csv = cfg.get(option = "children_csv", section = "myrpki", default = "children.csv") +pubclients_csv = cfg.get(option = "pubclients_csv", section = "myrpki", default = "pubclients.csv") +pubd_base = cfg.get(option = "pubd_base", section = "myirbe") + +# Here we need to construct values for the new config file from the +# old one. Basic model here is to look at whatever variables need to +# be set in the template (mostly just the [myrpki], I hope), pull +# necessary data from old config file any way we can. Stuff that +# didn't make the jump from old config file to new we can just ignore, +# stuff that is automated via macro expansions in the new config file +# should be ok without modification. + +r = {} + +if cfg.has_section("myrpki"): + for i in ("handle", "roa_csv", "prefix_csv", "asn_csv", "xml_filename"): + r["myrpki", i] = cfg.get(section = "myrpki", option = i) + r["myrpki", "bpki_resources_directory"] = cfg.get(option = "bpki_directory", section = "myrpki") + +if cfg.has_section("myirbe"): + r["myrpki", "bpki_servers_directory"] = cfg.get(option = "bpki_directory", section = "myirbe") + r["myrpki", "run_rpkid"] = True + r["myrpki", "run_pubd"] = cfg.getboolean(option = "want_pubd", section = "myirbe", default = False) + r["myrpki", "run_rootd"] = cfg.getboolean(option = "want_rootd", section = "myirbe", default = False) +else: + for i in ("run_rpkid", "run_pubd", "run_rootd"): + r["myrpki", i] = False + +if cfg.has_section("rpkid"): + r["myrpki", "rpkid_server_host"] = cfg.get(option = "server-host", section = "rpkid") + r["myrpki", "rpkid_server_port"] = cfg.get(option = "server-port", section = "rpkid") + +if cfg.has_section("irdbd"): + u = urlparse.urlparse(cfg.get(option = "https-url", section = "irdbd")) + r["myrpki", "irdbd_server_host"] = u.hostname or "localhost" + r["myrpki", "irdbd_server_port"] = u.port or 443 + +if cfg.has_section("pubd"): + r["myrpki", "pubd_server_host"] = cfg.get(option = "server-host", section = "pubd") + r["myrpki", "pubd_server_port"] = cfg.get(option = "server-port", section = "pubd") + r["myrpki", "publication_base_directory"] = cfg.get(option = "publication-base", section = "pubd") + +if cfg.has_section("rootd"): + r["myrpki", "rootd_server_port"] = cfg.get(option = "server-port", section = "rootd") + u = urlparse.urlparse(cfg.get(option = "rpki-base-uri", section = "rootd")) + r["myrpki", "publication_rsync_server"] = u.netloc + +for i in ("rpkid", "irdbd", "pubd"): + if cfg.has_section(i): + for j in ("sql-database", "sql-username", "sql-password"): + r[i, j] = cfg.get(section = i, option = j) + +f = open(new_cfg_file, "w") +f.write("# Automatically converted from %s using %s as a template.\n\n" % (cfg_file, template_file)) +section = None +for line in open(template_file): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + if m: + option, whitespace = m.group(1, 2) + else: + option = None + if (section, option) in r: + line = "%s%s%s\n" % (option, whitespace, r[section, option]) + f.write(line) +f.close() +print "Wrote", new_cfg_file + +# Get all of these from the new config file; in theory we just set all +# of them, but we want to use values matching new config in any case. + +newcfg = rpki.config.parser(new_cfg_file, "myrpki") + +handle = newcfg.get("handle") +bpki_resources_directory = newcfg.get("bpki_resources_directory") +bpki_servers_directory = newcfg.get("bpki_servers_directory") +pubd_server_host = newcfg.get("pubd_server_host") +pubd_server_port = newcfg.get("pubd_server_port") +rpkid_server_host = newcfg.get("rpkid_server_host") +rpkid_server_port = newcfg.get("rpkid_server_port") +entitydb_dir = newcfg.get("entitydb_dir", "entitydb") + +bpki_resources_pemfile = bpki_resources_directory + "/ca.cer" +bpki_servers_pemfile = bpki_servers_directory + "/ca.cer" + +def entitydb(*args): + return os.path.join(entitydb_dir, *args) + +# Now convert the .csv files. It'd be nice to have XML validation +# enabled for this, so try to turn it on ourselves if the magic +# environment variable hasn't already been set. + +rng_file = os.path.join(os.path.dirname(sys.argv[0]), "myrpki.rng") +if not os.getenv("MYRPKI_RNG") and os.path.exists(rng_file): + os.putenv("MYRPKI_RNG", rng_file) + +for d in map(entitydb, ("children", "parents", "repositories", "pubclients")): + if not os.path.exists(d): + os.makedirs(d) + +one_year_from_now = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365)) + +if os.path.exists(children_csv): + for child_handle, valid_until, child_resource_pemfile in myrpki.csv_reader(children_csv, columns = 3): + try: + + e = Element("parent", + valid_until = valid_until if preserve_valid_until else one_year_from_now, + service_uri = "https://%s:%s/up-down/%s/%s" % (rpkid_server_host, rpkid_server_port, handle, child_handle), + child_handle = child_handle, + parent_handle = handle) + myrpki.PEMElement(e, "bpki_resource_ta", bpki_resources_pemfile) + myrpki.PEMElement(e, "bpki_server_ta", bpki_servers_pemfile) + myrpki.PEMElement(e, "bpki_child_ta", child_resource_pemfile) + myrpki.etree_write(e, entitydb("children", "%s.xml" % child_handle)) + + except IOError: + pass + +if os.path.exists(parents_csv): + for parent_handle, parent_service_uri, parent_cms_pemfile, parent_https_pemfile, parent_myhandle, parent_sia_base in myrpki.csv_reader(parents_csv, columns = 6): + try: + + e = Element("parent", + valid_until = one_year_from_now, + service_uri = parent_service_uri, + child_handle = parent_myhandle, + parent_handle = parent_handle) + myrpki.PEMElement(e, "bpki_resource_ta", parent_cms_pemfile) + myrpki.PEMElement(e, "bpki_server_ta", parent_https_pemfile) + myrpki.PEMElement(e, "bpki_child_ta", bpki_resources_pemfile) + myrpki.etree_write(e, entitydb("parents", "%s.xml" % parent_handle)) + + client_handle = "/".join(parent_sia_base.rstrip("/").split("/")[3:]) + assert client_handle.startswith(repository_handle) + + e = Element("repository", + parent_handle = parent_handle, + client_handle = client_handle, + service_uri = "%s/client/%s" % (pubd_base.rstrip("/"), client_handle), + sia_base = parent_sia_base, + type = "confirmed") + myrpki.PEMElement(e, "bpki_server_ta", repository_bpki_certificate) + myrpki.PEMElement(e, "bpki_client_ta", bpki_resources_pemfile) + SubElement(e, "contact_info").text = "Automatically generated by convert-csv.py" + myrpki.etree_write(e, entitydb("repositories", "%s.xml" % parent_handle)) + + except IOError: + pass + +if os.path.exists(pubclients_csv): + for client_handle, client_resource_pemfile, client_sia_base in myrpki.csv_reader(pubclients_csv, columns = 3): + try: + + parent_handle = client_handle.split("/")[-2] if "/" in client_handle else handle + + e = Element("repository", + parent_handle = parent_handle, + client_handle = client_handle, + service_uri = "https://%s:%s/client/%s" % (pubd_server_host, pubd_server_port, client_handle), + sia_base = client_sia_base, + type = "confirmed") + myrpki.PEMElement(e, "bpki_server_ta", bpki_servers_pemfile) + myrpki.PEMElement(e, "bpki_client_ta", client_resource_pemfile) + SubElement(e, "contact_info").text = "Automatically generated by convert-csv.py" + myrpki.etree_write(e, entitydb("pubclients", "%s.xml" % client_handle.replace("/", "."))) + + except IOError: + pass diff --git a/myrpki/examples/asns.csv b/myrpki/examples/asns.csv new file mode 100644 index 00000000..804cf839 --- /dev/null +++ b/myrpki/examples/asns.csv @@ -0,0 +1,8 @@ +# $Id$ +# +# Syntax: <child_handle> <asn> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Alice 64533 diff --git a/myrpki/examples/myrpki.conf b/myrpki/examples/myrpki.conf new file mode 100644 index 00000000..24bcb2a7 --- /dev/null +++ b/myrpki/examples/myrpki.conf @@ -0,0 +1,460 @@ +# $Id: myrpki.conf 2722 2009-08-31 22:24:48Z sra $ +# +# Config file for myrpki.py, myirbe.py, and RPKI daemons when used +# with myrpki.py etc. +# +# NB: This config file is read both by Python code and also by the +# OpenSSL command line tool (running under mypki), so syntax must +# remain compatable with both parsers, and there's a big chunk of +# OpenSSL voodoo towards the end of this file. + +################################################################ + +[myrpki] + +# Handle naming hosted resource-holding entity (<self/>) represented +# by this myrpki instance. Syntax is an identifier (ASCII letters, +# digits, hyphen, underscore -- no whitespace, non-ASCII characters, +# or other punctuation). You need to set this. + +handle = Me + +# Names of various files and directories. Don't change these without +# a good reason. + +roa_csv = roas.csv +prefix_csv = prefixes.csv +asn_csv = asns.csv +xml_filename = myrpki.xml +bpki_resources_directory = bpki/resources +bpki_servers_directory = bpki/servers + +# Whether you want to run your own copy of rpkid (and irdbd). In +# general, if you're running myirbe.py at all, you want this on. + +run_rpkid = true + +# DNS hostname and server port numbers for rpkid and irdbd, if you're +# running them. rpkid's server host has to be a publicly reachable +# name to be useful; irdbd's server host should always be localhost +# unless you really know what you are doing. Port numbers can be any +# legal TCP port number that you're not using for something else. + +rpkid_server_host = rpkid.example.org +rpkid_server_port = 4404 +irdbd_server_host = localhost +irdbd_server_port = 4403 + +# Whether you want to run your own copy of pubd. In general, it's +# best to use your parent's pubd if you can, to reduce the overall +# number of publication sites that relying parties need to check, so +# don't enable this unless you have a good reason. + +run_pubd = true + +# DNS hostname and server port number for pubd, if you're running it. +# Hostname has to be a publicly reachable name to be useful, port can +# be any legal TCP port number that you're not using for something +# else. + +pubd_server_host = pubd.example.org +pubd_server_port = 4402 + +# Contact information to include in offers of repository service. +# This only matters when we're running pubd. This should be a human +# readable string, perhaps containing an email address or URL. + +pubd_contact_info = repo-man@rpki.example.org + +# Whether to offer repository service to our children. +# This only matters when we're running pubd. + +pubd_offer_service_to_children = false + +# Whether you want to run your very own copy of rootd. Don't enable +# this unless you really know what you're doing. + +run_rootd = false + +# Server port number for rootd, if you're running it. This can be any +# legal TCP port number that you're not using for something else. + +rootd_server_port = 4401 + +# Root of local directory tree where pubd (and rootd, sigh) should +# write out published data. You need to configure this, and the +# configuration should match up with the directory where you point +# rsyncd. Neither pubd nor rsyncd much cares -where- you tell them to +# put this stuff, the important thing is that the rsync:// URIs in +# generated certificates match up with the published objects so that +# relying parties can find and verify rpkid's published outputs. + +publication_base_directory = publication/ + +# rsyncd module name corresponding to publication_base_directory. +# This has to match the module you configured into rsyncd.conf. +# Leave this alone unless you have some need to change it. + +publication_rsync_module = rpki + +# Hostname and optional port number for rsync:// URIs. In most cases +# this should just be the same value as pubd_server_host. + +publication_rsync_server = ${myrpki::pubd_server_host} + +# SQL configuration. You can ignore this if you're not running any of +# the daemons yourself. + +# If you're comfortable with having all of the databases use the same +# MySQL username and password, set those values here. It's ok to +# leave the default username alone, but you should use a locally +# generated password either here or in the individual settings below. + +shared_sql_username = rpki +shared_sql_password = fnord + +# If you want different usernames and passwords for the separate SQL +# databases, enter those settings here; the shared_sql_* settings are +# only referenced here, so you can remove them entirely if you're +# setting everything in this block. + +rpkid_sql_database = rpkid +rpkid_sql_username = ${myrpki::shared_sql_username} +rpkid_sql_password = ${myrpki::shared_sql_password} + +irdbd_sql_database = irdbd +irdbd_sql_username = ${myrpki::shared_sql_username} +irdbd_sql_password = ${myrpki::shared_sql_password} + +pubd_sql_database = pubd +pubd_sql_username = ${myrpki::shared_sql_username} +pubd_sql_password = ${myrpki::shared_sql_password} + +# Name of OpenSSL binary. You might need to change this if you have +# no system copy installed, or if the system copy doesn't support CMS. +# The copy of openssl built by this package should suffice. + +openssl = openssl + +################################################################# + +# In theory it should not be necessary to modify anything below this +# point, at least not if you're within the boundaries of the +# simplified configuration that the myrpki tool is intended to +# support. If you do have to modify anything below this point, please +# report it. + +################################################################# + +[rpkid] + +# MySQL database name, user name, and password for rpkid to use to +# store its data. + +sql-database = ${myrpki::rpkid_sql_database} +sql-username = ${myrpki::rpkid_sql_username} +sql-password = ${myrpki::rpkid_sql_password} + +# Host and port on which rpkid should listen for HTTPS service +# requests. + +server-host = ${myrpki::rpkid_server_host} +server-port = ${myrpki::rpkid_server_port} + +# HTTPS service URL rpkid should use to contact irdbd. If irdbd is +# running on the same machine as rpkid, this can and probably should +# be a loopback URL, since nobody but rpkid needs to talk to irdbd. + +irdb-url = https://${myrpki::irdbd_server_host}:${myrpki::irdbd_server_port}/ + +# Where rpkid should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +rpkid-key = ${myrpki::bpki_servers_directory}/rpkid.key +rpkid-cert = ${myrpki::bpki_servers_directory}/rpkid.cer +irdb-cert = ${myrpki::bpki_servers_directory}/irdbd.cer +irbe-cert = ${myrpki::bpki_servers_directory}/irbe.cer + +################################################################# + +[irdbd] + +# MySQL database name, user name, and password for irdbd to use to +# store its data. + +sql-database = ${myrpki::irdbd_sql_database} +sql-username = ${myrpki::irdbd_sql_username} +sql-password = ${myrpki::irdbd_sql_password} + +# HTTP service URL irdbd should listen on. This should match the +# irdb-url parameter in the [rpkid] section; see comments there. + +https-url = https://${myrpki::irdbd_server_host}:${myrpki::irdbd_server_port}/ + +# Where irdbd should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +rpkid-cert = ${myrpki::bpki_servers_directory}/rpkid.cer +irdbd-cert = ${myrpki::bpki_servers_directory}/irdbd.cer +irdbd-key = ${myrpki::bpki_servers_directory}/irdbd.key + +################################################################# + +[pubd] + +# MySQL database name, user name, and password for pubd to use to +# store (some of) its data. + +sql-database = ${myrpki::pubd_sql_database} +sql-username = ${myrpki::pubd_sql_username} +sql-password = ${myrpki::pubd_sql_password} + +# Root of directory tree where pubd should write out published data. +# You need to configure this, and the configuration should match up +# with the directory where you point rsyncd. Neither pubd nor rsyncd +# much cares -where- you tell them to put this stuff, the important +# thing is that the rsync:// URIs in generated certificates match up +# with the published objects so that relying parties can find and +# verify rpkid's published outputs. + +publication-base = ${myrpki::publication_base_directory} + +# Host and port on which pubd should listen for HTTPS service +# requests. + +server-host = ${myrpki::pubd_server_host} +server-port = ${myrpki::pubd_server_port} + +# Where pubd should look for BPKI certs and keys used in the +# left-right protocol. The following values match where myirbe.py +# will have placed things. Don't change these without a reason. + +bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +pubd-cert = ${myrpki::bpki_servers_directory}/pubd.cer +pubd-key = ${myrpki::bpki_servers_directory}/pubd.key +irbe-cert = ${myrpki::bpki_servers_directory}/irbe.cer + +################################################################# + +[irbe_cli] + +# HTTPS service URL for rpkid + +rpkid-url = https://${myrpki::rpkid_server_host}:${myrpki::rpkid_server_port}/left-right/ + +# BPKI certificates and keys for talking to rpkid + +rpkid-bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +rpkid-irbe-key = ${myrpki::bpki_servers_directory}/irbe.key +rpkid-irbe-cert = ${myrpki::bpki_servers_directory}/irbe.cer +rpkid-cert = ${myrpki::bpki_servers_directory}/rpkid.cer + +# HTTPS service URL for pubd + +pubd-url = https://${myrpki::pubd_server_host}:${myrpki::pubd_server_port}/control/ + +# BPKI certificates and keys for talking to pubd + +pubd-bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +pubd-irbe-key = ${myrpki::bpki_servers_directory}/irbe.key +pubd-irbe-cert = ${myrpki::bpki_servers_directory}/irbe.cer +pubd-cert = ${myrpki::bpki_servers_directory}/pubd.cer + +################################################################# + +[rootd] + +# You don't need to run rootd unless you're IANA, are certifying +# private address space, or are an RIR which refuses to accept IANA as +# the root of the public address hierarchy. +# +# Ok, if that wasn't enough to scare you off: rootd is a kludge, and +# needs to be rewritten, or, better, merged into rpkid. It does a +# number of things wrong, and requires far too many configuration +# parameters. You have been warned.... + +# BPKI certificates and keys for rootd + +bpki-ta = ${myrpki::bpki_servers_directory}/ca.cer +rootd-bpki-crl = ${myrpki::bpki_servers_directory}/ca.crl +rootd-bpki-cert = ${myrpki::bpki_servers_directory}/rootd.cer +rootd-bpki-key = ${myrpki::bpki_servers_directory}/rootd.key +child-bpki-cert = ${myrpki::bpki_servers_directory}/child.cer + +# Server port on which rootd should listen. + +server-port = ${myrpki::rootd_server_port} + +# Where rootd should write its output. Yes, rootd should be using +# pubd instead of publishing directly, but it doesn't. + +rpki-root-dir = ${myrpki::publication_base_directory} + +# rsync URI for directory containing rootd's outputs + +rpki-base-uri = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/ + +# rsync URI for rootd's root (self-signed) RPKI certificate + +rpki-root-cert-uri = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/root.cer + +# Private key corresponding to rootd's root RPKI certificate + +rpki-root-key = ${myrpki::bpki_servers_directory}/ca.key + +# Filename (as opposed to rsync URI) of rootd's root RPKI certificate + +rpki-root-cert = ${myrpki::publication_base_directory}/root.cer + +# Where rootd should stash a copy of the PKCS #10 request it gets from +# its one (and only) child + +rpki-subject-pkcs10 = rootd.subject.pkcs10 + +# Lifetime of the one and only certificate rootd issues + +rpki-subject-lifetime = 30d + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the CRL +# for rootd's root RPKI certificate + +rpki-root-crl = root.crl + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the +# manifest for rootd's root RPKI certificate + +rpki-root-manifest = root.mnf + +# Up-down protocol class name for RPKI certificate rootd issues to its +# one (and only) child + +rpki-class-name = ${myrpki::handle} + +# Filename (relative to rootd-base-uri and rpki-root-dir) of the one +# (and only) RPKI certificate rootd issues + +rpki-subject-cert = ${myrpki::handle}.cer + +# The last four paramters in this section are really parameters for +# myirbe.py to use when constructing rootd's root RPKI certificate, +# via an indirection hack in the OpenSSL voodoo portion of this file. +# Don't ask why some of these are duplicated from other paramters in +# this section, you don't want to know (really, you don't). + +# ASNs to include in rootd's root RPKI certificate, in openssl.conf format + +root_cert_asns = AS:0-4294967295 + +# IP addresses to include in rootd's root RPKI certificate, in +# openssl.conf format + +root_cert_addrs = IPv4:0.0.0.0/0,IPv6:0::/0 + +# Whatever you put in rpki-base-uri, earlier in this section + +root_cert_sia = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/ + +# root_cert_sia + rpki-root-manifest + +root_cert_manifest = rsync://${myrpki::publication_rsync_server}/${myrpki::publication_rsync_module}/root.mnf + +################################################################# + +# Constants for OpenSSL voodoo portion of this file, to make them +# easier to find. + +[constants] + +# Digest algorithm. Don't change this. + +digest = sha256 + +# RSA key length. Don't change this. + +key_length = 2048 + +# Lifetime of BPKI certificates (and rootd RPKI root certificate). +# Don't change this unless you know what you're doing. + +cert_days = 365 + +# Lifetime of BPKI CRLs. Don't change this unless you know what +# you're doing. + +crl_days = 365 + +################################################################# + +# The rest of this file is OpenSSL configuration voodoo. Don't touch +# anything below here even if you -do- know what you're doing. Even +# by OpenSSL standards, some of this is weird, and interacts in +# non-obvious ways with code in myrpki.py and myirbe.py. If you touch +# this stuff and something breaks, don't say you weren't warned. + +[req] +default_bits = ${constants::key_length} +default_md = ${constants::digest} +distinguished_name = req_dn +prompt = no +encrypt_key = no + +[req_dn] +CN = Dummy name for certificate request + +[ca_x509_ext_ee] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_xcert0] +basicConstraints = critical,CA:true,pathlen:0 +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_xcert1] +basicConstraints = critical,CA:true,pathlen:1 +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca_x509_ext_ca] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[ca] +default_ca = ca +dir = ${ENV::BPKI_DIRECTORY} +new_certs_dir = $dir +database = $dir/index +certificate = $dir/ca.cer +private_key = $dir/ca.key +default_days = ${constants::cert_days} +default_crl_days = ${constants::crl_days} +default_md = ${constants::digest} +policy = ca_dn_policy +unique_subject = no +serial = $dir/serial +crlnumber = $dir/crl_number + +[ca_dn_policy] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional +givenName = optional +surname = optional + +[rootd_x509_extensions] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +keyUsage = critical,keyCertSign,cRLSign +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:${rootd::root_cert_sia},1.3.6.1.5.5.7.48.10;URI:${rootd::root_cert_manifest} +sbgp-autonomousSysNum = critical,${rootd::root_cert_asns} +sbgp-ipAddrBlock = critical,${rootd::root_cert_addrs} +certificatePolicies = critical,1.3.6.1.5.5.7.14.2 diff --git a/myrpki/examples/prefixes.csv b/myrpki/examples/prefixes.csv new file mode 100644 index 00000000..160f9339 --- /dev/null +++ b/myrpki/examples/prefixes.csv @@ -0,0 +1,11 @@ +# $Id$ +# +# Syntax: <child_handle> <prefix>/<length> +# or: <child_handle> <min>-<max> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +Alice 192.0.2.0/27 +Bob 192.0.2.44-192.0.2.100 +Bob 10.0.0.0/8 diff --git a/myrpki/examples/roas.csv b/myrpki/examples/roas.csv new file mode 100644 index 00000000..4343ada0 --- /dev/null +++ b/myrpki/examples/roas.csv @@ -0,0 +1,8 @@ +# $Id$ +# +# Syntax: <prefix>/<length>-<maxlength> <asn> <group> +# +# NB: Comment lines are not allowed in these files, this one is only +# present to explain the example +# +10.3.0.44/32 666 Mom diff --git a/myrpki/examples/rsyncd.conf b/myrpki/examples/rsyncd.conf new file mode 100644 index 00000000..fabb5aa2 --- /dev/null +++ b/myrpki/examples/rsyncd.conf @@ -0,0 +1,45 @@ +# $Id$ +# +# Sample rsyncd.conf file for use with pubd. You may need to +# customize this for the conventions on your system. See the rsync +# and rsyncd.conf manual pages for a complete explanation of how to +# configure rsyncd, this is just a simple configuration to get you +# started. +# +# There are two parameters in the following which you should set to +# appropriate values for your system: +# +# "myname" is the rsync module name to configure, as in +# "rsync://rpki.example.org/rpki/"; see the publication_rsync_module +# parameter in myrpki.conf +# +# "/some/where/publication" is the absolute pathname of the directory +# where you told pubd to place its outputs; see the +# publication_base_directory parameter in myrpki.conf. +# +# You may need to adjust other parameters for your system environment. +# +# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +pid file = /var/run/rsyncd.pid +uid = nobody +gid = nobody + +[rpki] + use chroot = no + read only = yes + transfer logging = yes + path = /some/where/publication + comment = RPKI Testbed diff --git a/myrpki/myrpki.py b/myrpki/myrpki.py new file mode 100644 index 00000000..a67ce15e --- /dev/null +++ b/myrpki/myrpki.py @@ -0,0 +1,1742 @@ +""" +This program is now the merger of three different tools: the old +myrpki.py script, the old myirbe.py script, and the newer setup.py CLI +tool. As such, it is still in need of some cleanup, but the need to +provide a saner user interface is more urgent than internal code +prettiness at the moment. In the long run, 90% of the code in this +file probably ought to move to well-designed library modules. + +Overall goal here is to build up the configuration necessary to run +rpkid and friends, by reading a config file, a collection of .CSV +files, and the results of a few out-of-band XML setup messages +exchanged with one's parents, children, and so forth. + +The config file is in an OpenSSL-compatible format, the CSV files are +simple tab-delimited text. The XML files are all generated by this +program, either the local instance or an instance being run by another +player in the system; the mechanism used to exchange these setup +messages is outside the scope of this program, feel free to use +PGP-signed mail, a web interface (not provided), USB stick, carrier +pigeons, whatever works. + +With one exception, the commands in this program avoid using any +third-party Python code other than the rpki libraries themselves; with +the same one exception, all OpenSSL work is done with the OpenSSL +command line tool (the one built as a side effect of building rcynic +will do, if your platform has no system copy or the system copy is too +old). This is all done in an attempt to make the code more portable, +so one can run most of the RPKI back end software on a laptop or +whatever. The one exception is the configure_daemons command, which +must, of necessity, use the same communication libraries as the +daemons with which it is conversing. So that one command will not +work if the correct Python modules are not available. + + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +from __future__ import with_statement + +import subprocess, csv, re, os, getopt, sys, base64, time, glob, copy, warnings +import rpki.config, rpki.cli, rpki.sundial, rpki.log, rpki.oids + +try: + from lxml.etree import (Element, SubElement, ElementTree, + fromstring as ElementFromString, + tostring as ElementToString) +except ImportError: + from xml.etree.ElementTree import (Element, SubElement, ElementTree, + fromstring as ElementFromString, + tostring as ElementToString) + + + +# Our XML namespace and protocol version. + +namespace = "http://www.hactrn.net/uris/rpki/myrpki/" +version = "2" +namespaceQName = "{" + namespace + "}" + +# Whether to include incomplete entries when rendering to XML. + +allow_incomplete = False + +# Whether to whine about incomplete entries while rendering to XML. + +whine = False + +class comma_set(set): + """ + Minor customization of set(), to provide a print syntax. + """ + + def __str__(self): + return ",".join(self) + +class EntityDB(object): + """ + Wrapper for entitydb path lookups. Hmm, maybe some or all of the + entitydb glob stuff should end up here too? Later. + """ + + def __init__(self, cfg): + self.dir = cfg.get("entitydb_dir", "entitydb") + + def __call__(self, *args): + return os.path.join(self.dir, *args) + + def iterate(self, *args): + return glob.iglob(os.path.join(self.dir, *args)) + +class roa_request(object): + """ + Representation of a ROA request. + """ + + v4re = re.compile("^([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+(-[0-9]+)?$", re.I) + v6re = re.compile("^([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+(-[0-9]+)?$", re.I) + + def __init__(self, asn, group): + self.asn = asn + self.group = group + self.v4 = comma_set() + self.v6 = comma_set() + + def __repr__(self): + s = "<%s asn %s group %s" % (self.__class__.__name__, self.asn, self.group) + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + return s + ">" + + def add(self, prefix): + """ + Add one prefix to this ROA request. + """ + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise RuntimeError, "Bad prefix syntax: %r" % (prefix,) + + def xml(self, e): + """ + Generate XML element represeting representing this ROA request. + """ + e = SubElement(e, "roa_request", + asn = self.asn, + v4 = str(self.v4), + v6 = str(self.v6)) + e.tail = "\n" + +class roa_requests(dict): + """ + Database of ROA requests. + """ + + def add(self, asn, group, prefix): + """ + Add one <ASN, group, prefix> set to ROA request database. + """ + key = (asn, group) + if key not in self: + self[key] = roa_request(asn, group) + self[key].add(prefix) + + def xml(self, e): + """ + Render ROA requests as XML elements. + """ + for r in self.itervalues(): + r.xml(e) + + @classmethod + def from_csv(cls, roa_csv_file): + """ + Parse ROA requests from CSV file. + """ + self = cls() + # format: p/n-m asn group + for pnm, asn, group in csv_reader(roa_csv_file, columns = 3): + self.add(asn = asn, group = group, prefix = pnm) + return self + +class child(object): + """ + Representation of one child entity. + """ + + v4re = re.compile("^(([0-9]{1,3}\.){3}[0-9]{1,3}/[0-9]+)|(([0-9]{1,3}\.){3}[0-9]{1,3}-([0-9]{1,3}\.){3}[0-9]{1,3})$", re.I) + v6re = re.compile("^(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}/[0-9]+)|(([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4}-([0-9a-f]{0,4}:){0,15}[0-9a-f]{0,4})$", re.I) + + def __init__(self, handle): + self.handle = handle + self.asns = comma_set() + self.v4 = comma_set() + self.v6 = comma_set() + self.validity = None + self.bpki_certificate = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.asns: + s += " asn %s" % self.asns + if self.v4: + s += " v4 %s" % self.v4 + if self.v6: + s += " v6 %s" % self.v6 + if self.validity: + s += " valid %s" % self.validity + if self.bpki_certificate: + s += " cert %s" % self.bpki_certificate + return s + ">" + + def add(self, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add prefix, autonomous system number, validity date, or BPKI + certificate for this child. + """ + if prefix is not None: + if self.v4re.match(prefix): + self.v4.add(prefix) + elif self.v6re.match(prefix): + self.v6.add(prefix) + else: + raise RuntimeError, "Bad prefix syntax: %r" % (prefix,) + if asn is not None: + self.asns.add(asn) + if validity is not None: + self.validity = validity + if bpki_certificate is not None: + self.bpki_certificate = bpki_certificate + + def xml(self, e): + """ + Render this child as an XML element. + """ + complete = self.bpki_certificate and self.validity + if whine and not complete: + print "Incomplete child entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "child", + handle = self.handle, + valid_until = self.validity, + asns = str(self.asns), + v4 = str(self.v4), + v6 = str(self.v6)) + e.tail = "\n" + if self.bpki_certificate: + PEMElement(e, "bpki_certificate", self.bpki_certificate) + +class children(dict): + """ + Database of children. + """ + + def add(self, handle, prefix = None, asn = None, validity = None, bpki_certificate = None): + """ + Add resources to a child, creating the child object if necessary. + """ + if handle not in self: + self[handle] = child(handle) + self[handle].add(prefix = prefix, asn = asn, validity = validity, bpki_certificate = bpki_certificate) + + def xml(self, e): + """ + Render children database to XML. + """ + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_csv(cls, prefix_csv_file, asn_csv_file, fxcert, entitydb): + """ + Parse child resources, certificates, and validity dates from CSV files. + """ + self = cls() + for f in entitydb.iterate("children", "*.xml"): + c = etree_read(f) + self.add(handle = os.path.splitext(os.path.split(f)[-1])[0], + validity = c.get("valid_until"), + bpki_certificate = fxcert(c.findtext("bpki_child_ta"))) + # childname p/n + for handle, pn in csv_reader(prefix_csv_file, columns = 2): + self.add(handle = handle, prefix = pn) + # childname asn + for handle, asn in csv_reader(asn_csv_file, columns = 2): + self.add(handle = handle, asn = asn) + return self + +class parent(object): + """ + Representation of one parent entity. + """ + + def __init__(self, handle): + self.handle = handle + self.service_uri = None + self.bpki_cms_certificate = None + self.bpki_https_certificate = None + self.myhandle = None + self.sia_base = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.myhandle: + s += " myhandle %s" % self.myhandle + if self.service_uri: + s += " uri %s" % self.service_uri + if self.sia_base: + s += " sia %s" % self.sia_base + if self.bpki_cms_certificate: + s += " cms %s" % self.bpki_cms_certificate + if self.bpki_https_certificate: + s += " https %s" % self.bpki_https_certificate + return s + ">" + + def add(self, service_uri = None, + bpki_cms_certificate = None, + bpki_https_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or BPKI certificates to this parent object. + """ + if service_uri is not None: + self.service_uri = service_uri + if bpki_cms_certificate is not None: + self.bpki_cms_certificate = bpki_cms_certificate + if bpki_https_certificate is not None: + self.bpki_https_certificate = bpki_https_certificate + if myhandle is not None: + self.myhandle = myhandle + if sia_base is not None: + self.sia_base = sia_base + + def xml(self, e): + """ + Render this parent object to XML. + """ + complete = self.bpki_cms_certificate and self.bpki_https_certificate and self.myhandle and self.service_uri and self.sia_base + if whine and not complete: + print "Incomplete parent entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "parent", + handle = self.handle, + myhandle = self.myhandle, + service_uri = self.service_uri, + sia_base = self.sia_base) + e.tail = "\n" + if self.bpki_cms_certificate: + PEMElement(e, "bpki_cms_certificate", self.bpki_cms_certificate) + if self.bpki_https_certificate: + PEMElement(e, "bpki_https_certificate", self.bpki_https_certificate) + +class parents(dict): + """ + Database of parent objects. + """ + + def add(self, handle, + service_uri = None, + bpki_cms_certificate = None, + bpki_https_certificate = None, + myhandle = None, + sia_base = None): + """ + Add service URI or certificates to parent object, creating it if necessary. + """ + if handle not in self: + self[handle] = parent(handle) + self[handle].add(service_uri = service_uri, + bpki_cms_certificate = bpki_cms_certificate, + bpki_https_certificate = bpki_https_certificate, + myhandle = myhandle, + sia_base = sia_base) + + def xml(self, e): + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_csv(cls, fxcert, entitydb): + """ + Parse parent data from entitydb. + """ + self = cls() + for f in entitydb.iterate("parents", "*.xml"): + h = os.path.splitext(os.path.split(f)[-1])[0] + p = etree_read(f) + r = etree_read(f.replace(os.path.sep + "parents" + os.path.sep, + os.path.sep + "repositories" + os.path.sep)) + assert r.get("type") == "confirmed" + self.add(handle = h, + service_uri = p.get("service_uri"), + bpki_cms_certificate = fxcert(p.findtext("bpki_resource_ta")), + bpki_https_certificate = fxcert(p.findtext("bpki_server_ta")), + myhandle = p.get("child_handle"), + sia_base = r.get("sia_base")) + return self + +class repository(object): + """ + Representation of one repository entity. + """ + + def __init__(self, handle): + self.handle = handle + self.service_uri = None + self.bpki_certificate = None + + def __repr__(self): + s = "<%s %s" % (self.__class__.__name__, self.handle) + if self.service_uri: + s += " uri %s" % self.service_uri + if self.bpki_certificate: + s += " cert %s" % self.bpki_certificate + return s + ">" + + def add(self, service_uri = None, bpki_certificate = None): + """ + Add service URI or BPKI certificates to this repository object. + """ + if service_uri is not None: + self.service_uri = service_uri + if bpki_certificate is not None: + self.bpki_certificate = bpki_certificate + + def xml(self, e): + """ + Render this repository object to XML. + """ + complete = self.bpki_certificate and self.service_uri + if whine and not complete: + print "Incomplete repository entry %s" % self + if complete or allow_incomplete: + e = SubElement(e, "repository", + handle = self.handle, + service_uri = self.service_uri) + e.tail = "\n" + if self.bpki_certificate: + PEMElement(e, "bpki_certificate", self.bpki_certificate) + +class repositories(dict): + """ + Database of repository objects. + """ + + def add(self, handle, + service_uri = None, + bpki_certificate = None): + """ + Add service URI or certificate to repository object, creating it if necessary. + """ + if handle not in self: + self[handle] = repository(handle) + self[handle].add(service_uri = service_uri, + bpki_certificate = bpki_certificate) + + def xml(self, e): + for c in self.itervalues(): + c.xml(e) + + @classmethod + def from_csv(cls, fxcert, entitydb): + """ + Parse repository data from entitydb. + """ + self = cls() + for f in entitydb.iterate("repositories", "*.xml"): + h = os.path.splitext(os.path.split(f)[-1])[0] + r = etree_read(f) + assert r.get("type") == "confirmed" + self.add(handle = h, + service_uri = r.get("service_uri"), + bpki_certificate = fxcert(r.findtext("bpki_server_ta"))) + return self + +class csv_reader(object): + """ + Reader for tab-delimited text that's (slightly) friendlier than the + stock Python csv module (which isn't intended for direct use by + humans anyway, and neither was this package originally, but that + seems to be the way that it has evolved...). + + Columns parameter specifies how many columns users of the reader + expect to see; lines with fewer columns will be padded with None + values. + + Original API design for this class courtesy of Warren Kumari, but + don't blame him if you don't like what I did with his ideas. + """ + + def __init__(self, filename, columns = None, min_columns = None, comment_characters = "#;"): + assert columns is None or isinstance(columns, int) + assert min_columns is None or isinstance(min_columns, int) + if columns is not None and min_columns is None: + min_columns = columns + self.filename = filename + self.columns = columns + self.min_columns = min_columns + self.comment_characters = comment_characters + self.file = open(filename, "r") + + def __iter__(self): + line_number = 0 + for line in self.file: + line_number += 1 + line = line.strip() + if not line or line[0] in self.comment_characters: + continue + fields = line.split() + if self.min_columns is not None and len(fields) < self.min_columns: + raise RuntimeError, "%s:%d: Not enough columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) > self.columns: + raise RuntimeError, "%s:%d: Too many columns in line %r" % (self.filename, line_number, line) + if self.columns is not None and len(fields) < self.columns: + fields += tuple(None for i in xrange(self.columns - len(fields))) + yield fields + +def csv_writer(filename): + """ + Writer object for tab delimited text. We just use the stock CSV + module in excel-tab mode for this. + """ + return csv.writer(open(filename, "w"), dialect = csv.get_dialect("excel-tab")) + + +def PEMElement(e, tag, filename, **kwargs): + """ + Create an XML element containing Base64 encoded data taken from a + PEM file. + """ + lines = open(filename).readlines() + while lines: + if lines.pop(0).startswith("-----BEGIN "): + break + while lines: + if lines.pop(-1).startswith("-----END "): + break + if e.text is None: + e.text = "\n" + se = SubElement(e, tag, **kwargs) + se.text = "\n" + "".join(lines) + se.tail = "\n" + return se + +class CA(object): + """ + Representation of one certification authority. + """ + + # Mapping of path restriction values we use to OpenSSL config file + # section names. + + path_restriction = { 0 : "ca_x509_ext_xcert0", + 1 : "ca_x509_ext_xcert1" } + + def __init__(self, cfg_file, dir): + self.cfg = cfg_file + self.dir = dir + self.cer = dir + "/ca.cer" + self.key = dir + "/ca.key" + self.req = dir + "/ca.req" + self.crl = dir + "/ca.crl" + self.index = dir + "/index" + self.serial = dir + "/serial" + self.crlnum = dir + "/crl_number" + + cfg = rpki.config.parser(cfg_file, "myrpki") + self.openssl = cfg.get("openssl", "openssl") + + self.env = { "PATH" : os.environ["PATH"], + "BPKI_DIRECTORY" : dir, + "RANDFILE" : ".OpenSSL.whines.unless.I.set.this", + "OPENSSL_CONF" : cfg_file } + + def run_openssl(self, *cmd, **kwargs): + """ + Run an OpenSSL command, suppresses stderr unless OpenSSL returns + failure, and returns stdout. + """ + stdin = kwargs.pop("stdin", None) + env = self.env.copy() + env.update(kwargs) + cmd = (self.openssl,) + cmd + p = subprocess.Popen(cmd, env = env, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.PIPE) + stdout, stderr = p.communicate(stdin) + if p.wait() != 0: + sys.stderr.write("OpenSSL command failed: " + stderr + "\n") + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + return stdout + + def run_ca(self, *args): + """ + Run OpenSSL "ca" command with common initial arguments. + """ + self.run_openssl("ca", "-batch", "-config", self.cfg, *args) + + def run_req(self, key_file, req_file, log_key = sys.stdout): + """ + Run OpenSSL "genrsa" and "req" commands. + """ + if not os.path.exists(key_file): + if log_key: + log_key.write("Generating 2048-bit RSA key %s\n" % os.path.realpath(key_file)) + self.run_openssl("genrsa", "-out", key_file, "2048") + if not os.path.exists(req_file): + self.run_openssl("req", "-new", "-sha256", "-config", self.cfg, "-key", key_file, "-out", req_file) + + def run_dgst(self, input, algorithm = "md5"): + """ + Run OpenSSL "dgst" command, return cleaned-up result. + """ + hash = self.run_openssl("dgst", "-" + algorithm, stdin = input) + # + # Twits just couldn't leave well enough alone, grr. + hash = "".join(hash.split()) + if hash.startswith("(stdin)="): + hash = hash[len("(stdin)="):] + return hash + + @staticmethod + def touch_file(filename, content = None): + """ + Create dumb little text files expected by OpenSSL "ca" utility. + """ + if not os.path.exists(filename): + f = open(filename, "w") + if content is not None: + f.write(content) + f.close() + + def setup(self, ca_name): + """ + Set up this CA. ca_name is an X.509 distinguished name in + /tag=val/tag=val format. + """ + + modified = False + + if not os.path.exists(self.dir): + os.makedirs(self.dir) + self.touch_file(self.index) + self.touch_file(self.serial, "01\n") + self.touch_file(self.crlnum, "01\n") + + self.run_req(key_file = self.key, req_file = self.req) + + if not os.path.exists(self.cer): + modified = True + self.run_ca("-selfsign", "-extensions", "ca_x509_ext_ca", "-subj", ca_name, "-in", self.req, "-out", self.cer) + + if not os.path.exists(self.crl): + modified = True + self.run_ca("-gencrl", "-out", self.crl) + + return modified + + def ee(self, ee_name, base_name): + """ + Issue an end-enity certificate. + """ + key_file = "%s/%s.key" % (self.dir, base_name) + req_file = "%s/%s.req" % (self.dir, base_name) + cer_file = "%s/%s.cer" % (self.dir, base_name) + self.run_req(key_file = key_file, req_file = req_file) + if not os.path.exists(cer_file): + self.run_ca("-extensions", "ca_x509_ext_ee", "-subj", ee_name, "-in", req_file, "-out", cer_file) + return True + else: + return False + + def cms_xml_sign(self, ee_name, base_name, elt): + """ + Sign an XML object with CMS, return Base64 text. + """ + self.ee(ee_name, base_name) + return base64.b64encode(self.run_openssl( + "cms", "-sign", "-binary", "-outform", "DER", + "-keyid", "-md", "sha256", "-nodetach", "-nosmimecap", + "-econtent_type", ".".join(str(i) for i in rpki.oids.name2oid["id-ct-xml"]), + "-inkey", "%s/%s.key" % (self.dir, base_name), + "-signer", "%s/%s.cer" % (self.dir, base_name), + stdin = ElementToString(etree_pre_write(elt)))) + + def cms_xml_verify(self, b64, ca): + """ + Attempt to verify and extract XML from a Base64-encoded signed CMS + object. CA is the filename of a certificate that we expect to be + the issuer of the EE certificate bundled with the CMS, and must + previously have been cross-certified under our trust anchor. + """ + # In theory, we should be able to use the -certfile parameter to + # pass in the CA certificate, but in practice, I have never gotten + # this to work, either with the command line tool or in the + # OpenSSL C API. Dunno why. Passing both TA and CA via -CAfile + # does work, so we do that, using a temporary file, sigh. + CAfile = os.path.join(self.dir, "temp.%s.pem" % os.getpid()) + try: + f = open(CAfile, "w") + f.write(open(self.cer).read()) + f.write(open(ca).read()) + f.close() + return etree_post_read(ElementFromString(self.run_openssl( + "cms", "-verify", "-inform", "DER", "-CAfile", CAfile, + stdin = base64.b64decode(b64)))) + finally: + if os.path.exists(CAfile): + os.unlink(CAfile) + + def bsc(self, pkcs10): + """ + Issue BSC certificiate, if we have a PKCS #10 request for it. + """ + + if pkcs10 is None: + return None, None + + pkcs10 = base64.b64decode(pkcs10) + + hash = self.run_dgst(pkcs10) + + req_file = "%s/bsc.%s.req" % (self.dir, hash) + cer_file = "%s/bsc.%s.cer" % (self.dir, hash) + + if not os.path.exists(cer_file): + self.run_openssl("req", "-inform", "DER", "-out", req_file, stdin = pkcs10) + self.run_ca("-extensions", "ca_x509_ext_ee", "-in", req_file, "-out", cer_file) + + return req_file, cer_file + + def fxcert(self, b64, filename = None, path_restriction = 0): + """ + Write PEM certificate to file, then cross-certify. + """ + fn = os.path.join(self.dir, filename or "temp.%s.cer" % os.getpid()) + try: + self.run_openssl("x509", "-inform", "DER", "-out", fn, + stdin = base64.b64decode(b64)) + return self.xcert(fn, path_restriction) + finally: + if not filename and os.path.exists(fn): + os.unlink(fn) + pass + + def xcert(self, cert, path_restriction = 0): + """ + Cross-certify a certificate represented as a PEM file. + """ + + if not cert or not os.path.exists(cert): + return None + + # Extract public key and subject name from PEM file and hash it so + # we can use the result as a tag for cross-certifying this cert. + + hash = self.run_dgst(self.run_openssl( + "x509", "-noout", "-pubkey", "-subject", "-in", cert)) + + # Cross-certify the cert we were given, if we haven't already. + # This only works for self-signed certs, due to limitations of the + # OpenSSL command line tool, but that suffices for our purposes. + + xcert = "%s/xcert.%s.cer" % (self.dir, hash.strip()) + if not os.path.exists(xcert): + self.run_ca("-ss_cert", cert, "-out", xcert, "-extensions", self.path_restriction[path_restriction]) + return xcert + +def etree_validate(e): + # This is a kludge, schema should be loaded as module or configured + # in .conf, but it will do as a temporary debugging hack. + schema = os.getenv("MYRPKI_RNG") + if schema: + try: + import lxml.etree + except ImportError: + return + try: + lxml.etree.RelaxNG(file = schema).assertValid(e) + except lxml.etree.RelaxNGParseError: + return + except lxml.etree.DocumentInvalid: + print lxml.etree.tostring(e, pretty_print = True) + raise + +def etree_write(e, filename, verbose = False, validate = True, msg = None): + """ + Write out an etree to a file, safely. + + I still miss SYSCAL(RENMWO). + """ + filename = os.path.realpath(filename) + tempname = filename + if not filename.startswith("/dev/"): + tempname += ".tmp" + if verbose or msg: + print "Writing", filename + if msg: + print msg + e = etree_pre_write(e, validate) + ElementTree(e).write(tempname) + if tempname != filename: + os.rename(tempname, filename) + +def etree_pre_write(e, validate = True): + """ + Do the namespace frobbing needed on write; broken out of + etree_write() because also needed with ElementToString(). + """ + e = copy.deepcopy(e) + e.set("version", version) + for i in e.getiterator(): + if i.tag[0] != "{": + i.tag = namespaceQName + i.tag + assert i.tag.startswith(namespaceQName) + if validate: + etree_validate(e) + return e + +def etree_read(filename, verbose = False, validate = True): + """ + Read an etree from a file, verifying then stripping XML namespace + cruft. + """ + if verbose: + print "Reading", filename + e = ElementTree(file = filename).getroot() + return etree_post_read(e, validate) + +def etree_post_read(e, validate = True): + """ + Do the namespace frobbing needed on read; broken out of etree_read() + beause also needed by ElementFromString(). + """ + if validate: + etree_validate(e) + for i in e.getiterator(): + if i.tag.startswith(namespaceQName): + i.tag = i.tag[len(namespaceQName):] + else: + raise RuntimeError, "XML tag %r is not in namespace %r" % (i.tag, namespace) + return e + +def b64_equal(thing1, thing2): + """ + Compare two Base64-encoded values for equality. + """ + return "".join(thing1.split()) == "".join(thing2.split()) + + + +class main(rpki.cli.Cmd): + + prompt = "myrpki> " + + completedefault = rpki.cli.Cmd.filename_complete + + show_xml = False + + def __init__(self): + os.environ["TZ"] = "UTC" + time.tzset() + + rpki.log.use_syslog = False + + self.cfg_file = os.getenv("MYRPKI_CONF", "myrpki.conf") + + opts, argv = getopt.getopt(sys.argv[1:], "c:h?", ["config=", "help"]) + for o, a in opts: + if o in ("-c", "--config"): + self.cfg_file = a + elif o in ("-h", "--help", "-?"): + argv = ["help"] + + if not argv or argv[0] != "help": + rpki.log.init("myrpki") + self.read_config() + + rpki.cli.Cmd.__init__(self, argv) + + + def help_overview(self): + """ + Show program __doc__ string. Perhaps there's some clever way to + do this using the textwrap module, but for now something simple + and crude will suffice. + """ + for line in __doc__.splitlines(True): + self.stdout.write(" " * 4 + line) + self.stdout.write("\n") + + def read_config(self): + + self.cfg = rpki.config.parser(self.cfg_file, "myrpki") + + self.histfile = self.cfg.get("history_file", ".myrpki_history") + self.handle = self.cfg.get("handle") + self.run_rpkid = self.cfg.getboolean("run_rpkid") + self.run_pubd = self.cfg.getboolean("run_pubd") + self.run_rootd = self.cfg.getboolean("run_rootd") + self.entitydb = EntityDB(self.cfg) + + if self.run_rootd and (not self.run_pubd or not self.run_rpkid): + raise RuntimeError, "Can't run rootd unless also running rpkid and pubd" + + self.bpki_resources = CA(self.cfg_file, self.cfg.get("bpki_resources_directory")) + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.bpki_servers = CA(self.cfg_file, self.cfg.get("bpki_servers_directory")) + + self.pubd_contact_info = self.cfg.get("pubd_contact_info", "") + + self.rsync_module = self.cfg.get("publication_rsync_module") + self.rsync_server = self.cfg.get("publication_rsync_server") + + + def do_initialize(self, arg): + """ + Initialize an RPKI installation. This command reads the + configuration file, creates the BPKI and EntityDB directories, + generates the initial BPKI certificates, and creates an XML file + describing the resource-holding aspect of this RPKI installation. + """ + + if arg: + raise RuntimeError, "This command takes no arguments" + + self.bpki_resources.setup(self.cfg.get("bpki_resources_ta_dn", + "/CN=%s BPKI Resource Trust Anchor" % self.handle)) + if self.run_rpkid or self.run_pubd or self.run_rootd: + self.bpki_servers.setup(self.cfg.get("bpki_servers_ta_dn", + "/CN=%s BPKI Server Trust Anchor" % self.handle)) + + # Create entitydb directories. + + for i in ("parents", "children", "repositories", "pubclients"): + d = self.entitydb(i) + if not os.path.exists(d): + os.makedirs(d) + + if self.run_rpkid or self.run_pubd or self.run_rootd: + + if self.run_rpkid: + self.bpki_servers.ee(self.cfg.get("bpki_rpkid_ee_dn", + "/CN=%s rpkid server certificate" % self.handle), "rpkid") + self.bpki_servers.ee(self.cfg.get("bpki_irdbd_ee_dn", + "/CN=%s irdbd server certificate" % self.handle), "irdbd") + if self.run_pubd: + self.bpki_servers.ee(self.cfg.get("bpki_pubd_ee_dn", + "/CN=%s pubd server certificate" % self.handle), "pubd") + if self.run_rpkid or self.run_pubd: + self.bpki_servers.ee(self.cfg.get("bpki_irbe_ee_dn", + "/CN=%s irbe client certificate" % self.handle), "irbe") + if self.run_rootd: + self.bpki_servers.ee(self.cfg.get("bpki_rootd_ee_dn", + "/CN=%s rootd server certificate" % self.handle), "rootd") + + # Build the identity.xml file. Need to check for existing file so we don't + # overwrite? Worry about that later. + + e = Element("identity", handle = self.handle) + PEMElement(e, "bpki_ta", self.bpki_resources.cer) + etree_write(e, self.entitydb("identity.xml"), + msg = None if self.run_rootd else 'This is the "identity" file you will need to send to your parent') + + # If we're running rootd, construct a fake parent to go with it, + # and cross-certify in both directions so we can talk to rootd. + + if self.run_rootd: + + e = Element("parent", parent_handle = self.handle, child_handle = self.handle, + service_uri = "https://localhost:%s/" % self.cfg.get("rootd_server_port"), + valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365))) + PEMElement(e, "bpki_resource_ta", self.bpki_servers.cer) + PEMElement(e, "bpki_server_ta", self.bpki_servers.cer) + PEMElement(e, "bpki_child_ta", self.bpki_resources.cer) + SubElement(e, "repository", type = "offer") + etree_write(e, self.entitydb("parents", "%s.xml" % self.handle)) + + self.bpki_resources.xcert(self.bpki_servers.cer) + + rootd_child_fn = self.cfg.get("child-bpki-cert", None, "rootd") + if not os.path.exists(rootd_child_fn): + os.link(self.bpki_servers.xcert(self.bpki_resources.cer), rootd_child_fn) + + repo_file_name = self.entitydb("repositories", "%s.xml" % self.handle) + + try: + want_offer = etree_read(repo_file_name).get("type") != "confirmed" + except IOError: + want_offer = True + + if want_offer: + e = Element("repository", type = "offer", handle = self.handle, parent_handle = self.handle) + PEMElement(e, "bpki_client_ta", self.bpki_resources.cer) + etree_write(e, repo_file_name, + msg = 'This is the "repository offer" file for you to use if you want to publish in your own repository') + + def do_configure_child(self, arg): + """ + Configure a new child of this RPKI entity, given the child's XML + identity file as an input. This command extracts the child's data + from the XML, cross-certifies the child's resource-holding BPKI + certificate, and generates an XML file describing the relationship + between the child and this parent, including this parent's BPKI + data and up-down protocol service URI. + """ + + child_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["child_handle="]) + for o, a in opts: + if o == "--child_handle": + child_handle = a + + if len(argv) != 1: + raise RuntimeError, "Need to specify filename for child.xml" + + c = etree_read(argv[0]) + + if child_handle is None: + child_handle = c.get("handle") + + try: + e = etree_read(self.cfg.get("xml_filename")) + service_uri_base = e.get("service_uri") + except IOError: + service_uri_base = None + + if not service_uri_base and self.run_rpkid: + service_uri_base = "https://%s:%s/up-down/%s" % (self.cfg.get("rpkid_server_host"), + self.cfg.get("rpkid_server_port"), + self.handle) + if not service_uri_base: + print "Sorry, you can't set up children of a hosted config that itself has not yet been set up" + return + + print "Child calls itself %r, we call it %r" % (c.get("handle"), child_handle) + + self.bpki_servers.fxcert(c.findtext("bpki_ta")) + + e = Element("parent", parent_handle = self.handle, child_handle = child_handle, + service_uri = "%s/%s" % (service_uri_base, child_handle), + valid_until = str(rpki.sundial.now() + rpki.sundial.timedelta(days = 365))) + + PEMElement(e, "bpki_resource_ta", self.bpki_resources.cer) + PEMElement(e, "bpki_server_ta", self.bpki_servers.cer) + SubElement(e, "bpki_child_ta").text = c.findtext("bpki_ta") + + try: + repo = None + for f in self.entitydb.iterate("repositories", "*.xml"): + r = etree_read(f) + if r.get("type") == "confirmed": + if repo is not None: + raise RuntimeError, "Too many repositories, I don't know what to do, not giving referral" + repo_handle = os.path.splitext(os.path.split(f)[-1])[0] + repo = r + if repo is None: + raise RuntimeError, "Couldn't find any usable repositories, not giving referral" + + if repo_handle == self.handle: + SubElement(e, "repository", type = "offer") + else: + proposed_sia_base = repo.get("sia_base") + child_handle + "/" + r = Element("referral", authorized_sia_base = proposed_sia_base) + r.text = c.findtext("bpki_ta") + auth = self.bpki_resources.cms_xml_sign( + "/CN=%s Publication Referral" % self.handle, "referral", r) + r = SubElement(e, "repository", type = "referral") + SubElement(r, "authorization", referrer = repo.get("client_handle")).text = auth + SubElement(r, "contact_info").text = repo.findtext("contact_info") + + except RuntimeError, err: + print err + + etree_write(e, self.entitydb("children", "%s.xml" % child_handle), + msg = "Send this file back to the child you just configured") + + + def do_configure_parent(self, arg): + """ + Configure a new parent of this RPKI entity, given the output of + the parent's configure_child command as input. This command reads + the parent's response XML, extracts the parent's BPKI and service + URI information, cross-certifies the parent's BPKI data into this + entity's BPKI, and checks for offers or referrals of publication + service. If a publication offer or referral is present, we + generate a request-for-service message to that repository, in case + the user wants to avail herself of the referral or offer. + """ + + parent_handle = None + + opts, argv = getopt.getopt(arg.split(), "", ["parent_handle="]) + for o, a in opts: + if o == "--parent_handle": + parent_handle = a + + if len(argv) != 1: + raise RuntimeError, "Need to specify filename for parent.xml on command line" + + p = etree_read(argv[0]) + + if parent_handle is None: + parent_handle = p.get("parent_handle") + + print "Parent calls itself %r, we call it %r" % (p.get("parent_handle"), parent_handle) + print "Parent calls us %r" % p.get("child_handle") + + self.bpki_resources.fxcert(p.findtext("bpki_resource_ta")) + self.bpki_resources.fxcert(p.findtext("bpki_server_ta")) + + etree_write(p, self.entitydb("parents", "%s.xml" % parent_handle)) + + r = p.find("repository") + + if r is not None and r.get("type") in ("offer", "referral"): + r.set("handle", self.handle) + r.set("parent_handle", parent_handle) + PEMElement(r, "bpki_client_ta", self.bpki_resources.cer) + etree_write(r, self.entitydb("repositories", "%s.xml" % parent_handle), + msg = 'This is the "repository %s" file to send to the repository operator' % r.get("type")) + else: + print "Couldn't find repository offer or referral" + + + def do_configure_publication_client(self, arg): + """ + Configure publication server to know about a new client, given the + client's request-for-service message as input. This command reads + the client's request for service, cross-certifies the client's + BPKI data, and generates a response message containing the + repository's BPKI data and service URI. + """ + + sia_base = None + + opts, argv = getopt.getopt(arg.split(), "", ["sia_base="]) + for o, a in opts: + if o == "--sia_base": + sia_base = a + + if len(argv) != 1: + raise RuntimeError, "Need to specify filename for client.xml" + + client = etree_read(argv[0]) + + if sia_base is None: + + auth = client.find("authorization") + if auth is not None: + print "Found <authorization/> element, this looks like a referral" + referrer = etree_read(self.entitydb("pubclients", "%s.xml" % auth.get("referrer").replace("/","."))) + referrer = self.bpki_servers.fxcert(referrer.findtext("bpki_client_ta")) + referral = self.bpki_servers.cms_xml_verify(auth.text, referrer) + if not b64_equal(referral.text, client.findtext("bpki_client_ta")): + raise RuntimeError, "Referral trust anchor does not match" + sia_base = referral.get("authorized_sia_base") + + elif client.get("parent_handle") == self.handle: + print "Client claims to be our child, checking" + client_ta = client.findtext("bpki_client_ta") + assert client_ta + for child in self.entitydb.iterate("children", "*.xml"): + c = etree_read(child) + if b64_equal(c.findtext("bpki_child_ta"), client_ta): + sia_base = "rsync://%s/%s/%s/%s/" % (self.rsync_server, self.rsync_module, + self.handle, client.get("handle")) + break + + # If we still haven't figured out what to do with this client, it + # gets a top-level tree of its own, no attempt at nesting. + + if sia_base is None: + print "Don't know where to nest this client, defaulting to top-level" + sia_base = "rsync://%s/%s/%s/" % (self.rsync_server, self.rsync_module, client.get("handle")) + + assert sia_base.startswith("rsync://") + + client_handle = "/".join(sia_base.rstrip("/").split("/")[4:]) + + parent_handle = client.get("parent_handle") + + print "Client calls itself %r, we call it %r" % (client.get("handle"), client_handle) + print "Client says its parent handle is %r" % parent_handle + + self.bpki_servers.fxcert(client.findtext("bpki_client_ta")) + + e = Element("repository", type = "confirmed", + client_handle = client_handle, + parent_handle = parent_handle, + sia_base = sia_base, + service_uri = "https://%s:%s/client/%s" % (self.cfg.get("pubd_server_host"), + self.cfg.get("pubd_server_port"), + client_handle)) + + PEMElement(e, "bpki_server_ta", self.bpki_servers.cer) + SubElement(e, "bpki_client_ta").text = client.findtext("bpki_client_ta") + SubElement(e, "contact_info").text = self.pubd_contact_info + etree_write(e, self.entitydb("pubclients", "%s.xml" % client_handle.replace("/", ".")), + msg = "Send this file back to the publication client you just configured") + + + def do_configure_repository(self, arg): + """ + Configure a publication repository for this RPKI entity, given the + repository's response to our request-for-service message as input. + This command reads the repository's response, extracts and + cross-certifies the BPKI data and service URI, and links the + repository data with the corresponding parent data in our local + database. + """ + + argv = arg.split() + + if len(argv) != 1: + raise RuntimeError, "Need to specify filename for repository.xml on command line" + + r = etree_read(argv[0]) + + parent_handle = r.get("parent_handle") + + print "Repository calls us %r" % (r.get("client_handle")) + print "Repository response associated with parent_handle %r" % parent_handle + + etree_write(r, self.entitydb("repositories", "%s.xml" % parent_handle)) + + + + + def configure_resources_main(self, msg = None): + """ + Main program of old myrpki.py script. This remains separate + because it's called from more than one place. + """ + + roa_csv_file = self.cfg.get("roa_csv") + prefix_csv_file = self.cfg.get("prefix_csv") + asn_csv_file = self.cfg.get("asn_csv") + + # This probably should become an argument instead of (or in + # addition to a default from?) a config file option. + xml_filename = self.cfg.get("xml_filename") + + try: + e = etree_read(xml_filename) + bsc_req, bsc_cer = self.bpki_resources.bsc(e.findtext("bpki_bsc_pkcs10")) + service_uri = e.get("service_uri") + server_ta = e.findtext("bpki_server_ta") + except IOError: + bsc_req, bsc_cer = None, None + service_uri = None + server_ta = None + + e = Element("myrpki", handle = self.handle) + + if service_uri: + e.set("service_uri", service_uri) + + roa_requests.from_csv(roa_csv_file).xml(e) + + children.from_csv( + prefix_csv_file = prefix_csv_file, + asn_csv_file = asn_csv_file, + fxcert = self.bpki_resources.fxcert, + entitydb = self.entitydb).xml(e) + + parents.from_csv( fxcert = self.bpki_resources.fxcert, entitydb = self.entitydb).xml(e) + repositories.from_csv(fxcert = self.bpki_resources.fxcert, entitydb = self.entitydb).xml(e) + + PEMElement(e, "bpki_ca_certificate", self.bpki_resources.cer) + PEMElement(e, "bpki_crl", self.bpki_resources.crl) + + if bsc_cer: + PEMElement(e, "bpki_bsc_certificate", bsc_cer) + + if bsc_req: + PEMElement(e, "bpki_bsc_pkcs10", bsc_req) + + if server_ta: + SubElement(e, "bpki_server_ta").text = server_ta + + etree_write(e, xml_filename, msg = msg) + + + def do_configure_resources(self, arg): + """ + Read CSV files and all the descriptions of parents and children + that we've built up, package the result up as a single XML file to + be shipped to a hosting rpkid. + """ + + if arg: + raise RuntimeError, "Unexpected argument %r" % arg + self.configure_resources_main(msg = "Send this file to the rpkid operator who is hosting you") + + + + def do_configure_daemons(self, arg): + """ + Configure RPKI daemons with the data built up by the other + commands in this program. + + The basic model here is that each entity with resources to certify + runs the myrpki tool, but not all of them necessarily run their + own RPKI engines. The entities that do run RPKI engines get data + from the entities they host via the XML files output by the + configure_resources command. Those XML files are the input to + this command, which uses them to do all the work of configuring + daemons, populating SQL databases, and so forth. A few operations + (eg, BSC construction) generate data which has to be shipped back + to the resource holder, which we do by updating the same XML file. + + In essence, the XML files are a sneakernet (or email, or carrier + pigeon) communication channel between the resource holders and the + RPKI engine operators. + + As a convenience, for the normal case where the RPKI engine + operator is itself a resource holder, this command in effect runs + the configure_resources command automatically to process the RPKI + engine operator's own resources. + + Note that, due to the back and forth nature of some of these + operations, it may take several cycles for data structures to stablize + and everything to reach a steady state. This is normal. + """ + + argv = arg.split() + + try: + import rpki.https, rpki.resource_set, rpki.relaxng, rpki.exceptions + import rpki.left_right, rpki.x509, rpki.async, lxml.etree + if hasattr(warnings, "catch_warnings"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import MySQLdb + else: + import MySQLdb + + except ImportError, e: + print "Sorry, you appear to be missing some of the Python modules needed to run this command" + print "[Error: %r]" % e + + def findbase64(tree, name, b64type = rpki.x509.X509): + x = tree.findtext(name) + return b64type(Base64 = x) if x else None + + # We can use a single BSC for everything -- except BSC key + # rollovers. Drive off that bridge when we get to it. + + bsc_handle = "bsc" + + self.cfg.set_global_flags() + + # Default values for CRL parameters are low, for testing. Not + # quite as low as they once were, too much expired CRL whining. + + self_crl_interval = self.cfg.getint("self_crl_interval", 2 * 60 * 60) + self_regen_margin = self.cfg.getint("self_regen_margin", 30 * 60) + pubd_base = "https://%s:%s/" % (self.cfg.get("pubd_server_host"), self.cfg.get("pubd_server_port")) + rpkid_base = "https://%s:%s/" % (self.cfg.get("rpkid_server_host"), self.cfg.get("rpkid_server_port")) + + # Wrappers to simplify calling rpkid and pubd. + + call_rpkid = rpki.async.sync_wrapper(rpki.https.caller( + proto = rpki.left_right, + client_key = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = self.bpki_servers.cer), + server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/rpkid.cer"), + url = rpkid_base + "left-right", + debug = self.show_xml)) + + if self.run_pubd: + + call_pubd = rpki.async.sync_wrapper(rpki.https.caller( + proto = rpki.publication, + client_key = rpki.x509.RSA( PEM_file = self.bpki_servers.dir + "/irbe.key"), + client_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/irbe.cer"), + server_ta = rpki.x509.X509(PEM_file = self.bpki_servers.cer), + server_cert = rpki.x509.X509(PEM_file = self.bpki_servers.dir + "/pubd.cer"), + url = pubd_base + "control", + debug = self.show_xml)) + + # Make sure that pubd's BPKI CRL is up to date. + + call_pubd(rpki.publication.config_elt.make_pdu( + action = "set", + bpki_crl = rpki.x509.CRL(PEM_file = self.bpki_servers.crl))) + + irdbd_cfg = rpki.config.parser(self.cfg.get("irdbd_conf", self.cfg_file), "irdbd") + + db = MySQLdb.connect(user = irdbd_cfg.get("sql-username"), + db = irdbd_cfg.get("sql-database"), + passwd = irdbd_cfg.get("sql-password")) + + cur = db.cursor() + + xmlfiles = [] + + # If [myrpki] section includes an "xml_filename" setting, run + # myrpki.py internally, as a convenience, and include its output at + # the head of our list of XML files to process. + + my_xmlfile = self.cfg.get("xml_filename", "") + if my_xmlfile: + self.configure_resources_main() + xmlfiles.append(my_xmlfile) + else: + my_xmlfile = None + + # Add any other XML files specified on the command line + + xmlfiles.extend(argv) + + my_handle = None + + for xmlfile in xmlfiles: + + # Parse XML file and validate it against our scheme + + tree = etree_read(xmlfile, validate = True) + + handle = tree.get("handle") + + if xmlfile == my_xmlfile: + my_handle = handle + + # Update IRDB with parsed resource and roa-request data. + + cur.execute( + """ + DELETE + FROM roa_request_prefix + USING roa_request, roa_request_prefix + WHERE roa_request.roa_request_id = roa_request_prefix.roa_request_id AND roa_request.roa_request_handle = %s + """, (handle,)) + + cur.execute("DELETE FROM roa_request WHERE roa_request.roa_request_handle = %s", (handle,)) + + for x in tree.getiterator("roa_request"): + cur.execute("INSERT roa_request (roa_request_handle, asn) VALUES (%s, %s)", (handle, x.get("asn"))) + roa_request_id = cur.lastrowid + for version, prefix_set in ((4, rpki.resource_set.roa_prefix_set_ipv4(x.get("v4"))), (6, rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")))): + if prefix_set: + cur.executemany("INSERT roa_request_prefix (roa_request_id, prefix, prefixlen, max_prefixlen, version) VALUES (%s, %s, %s, %s, %s)", + ((roa_request_id, p.prefix, p.prefixlen, p.max_prefixlen, version) for p in prefix_set)) + + cur.execute( + """ + DELETE + FROM registrant_asn + USING registrant, registrant_asn + WHERE registrant.registrant_id = registrant_asn.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute( + """ + DELETE FROM registrant_net USING registrant, registrant_net + WHERE registrant.registrant_id = registrant_net.registrant_id AND registrant.registry_handle = %s + """ , (handle,)) + + cur.execute("DELETE FROM registrant WHERE registrant.registry_handle = %s" , (handle,)) + + for x in tree.getiterator("child"): + child_handle = x.get("handle") + asns = rpki.resource_set.resource_set_as(x.get("asns")) + ipv4 = rpki.resource_set.resource_set_ipv4(x.get("v4")) + ipv6 = rpki.resource_set.resource_set_ipv6(x.get("v6")) + + cur.execute("INSERT registrant (registrant_handle, registry_handle, registrant_name, valid_until) VALUES (%s, %s, %s, %s)", + (child_handle, handle, child_handle, rpki.sundial.datetime.fromXMLtime(x.get("valid_until")).to_sql())) + child_id = cur.lastrowid + if asns: + cur.executemany("INSERT registrant_asn (start_as, end_as, registrant_id) VALUES (%s, %s, %s)", + ((a.min, a.max, child_id) for a in asns)) + if ipv4: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 4, %s)", + ((a.min, a.max, child_id) for a in ipv4)) + if ipv6: + cur.executemany("INSERT registrant_net (start_ip, end_ip, version, registrant_id) VALUES (%s, %s, 6, %s)", + ((a.min, a.max, child_id) for a in ipv6)) + + db.commit() + + # Check for certificates before attempting anything else + + hosted_cacert = findbase64(tree, "bpki_ca_certificate") + if not hosted_cacert: + print "Nothing else I can do without a trust anchor for the entity I'm hosting." + continue + + rpkid_xcert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(b64 = hosted_cacert.get_Base64(), + filename = handle + ".cacert.cer", + path_restriction = 1)) + + # See what rpkid and pubd already have on file for this entity. + + if self.run_pubd: + client_pdus = dict((x.client_handle, x) + for x in call_pubd(rpki.publication.client_elt.make_pdu(action = "list")) + if isinstance(x, rpki.publication.client_elt)) + + rpkid_reply = call_rpkid( + rpki.left_right.self_elt.make_pdu( action = "get", tag = "self", self_handle = handle), + rpki.left_right.bsc_elt.make_pdu( action = "list", tag = "bsc", self_handle = handle), + rpki.left_right.repository_elt.make_pdu(action = "list", tag = "repository", self_handle = handle), + rpki.left_right.parent_elt.make_pdu( action = "list", tag = "parent", self_handle = handle), + rpki.left_right.child_elt.make_pdu( action = "list", tag = "child", self_handle = handle)) + + self_pdu = rpkid_reply[0] + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + repository_pdus = dict((x.repository_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.repository_elt)) + parent_pdus = dict((x.parent_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.parent_elt)) + child_pdus = dict((x.child_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.child_elt)) + + pubd_query = [] + rpkid_query = [] + + # There should be exactly one <self/> object per hosted entity, by definition + + if (isinstance(self_pdu, rpki.left_right.report_error_elt) or + self_pdu.crl_interval != self_crl_interval or + self_pdu.regen_margin != self_regen_margin or + self_pdu.bpki_cert != rpkid_xcert): + rpkid_query.append(rpki.left_right.self_elt.make_pdu( + action = "create" if isinstance(self_pdu, rpki.left_right.report_error_elt) else "set", + tag = "self", + self_handle = handle, + bpki_cert = rpkid_xcert, + crl_interval = self_crl_interval, + regen_margin = self_regen_margin)) + + # In general we only need one <bsc/> per <self/>. BSC objects are a + # little unusual in that the PKCS #10 subelement is generated by rpkid + # in response to generate_keypair, so there's more of a separation + # between create and set than with other objects. + + bsc_cert = findbase64(tree, "bpki_bsc_certificate") + bsc_crl = findbase64(tree, "bpki_crl", rpki.x509.CRL) + + bsc_pdu = bsc_pdus.pop(bsc_handle, None) + + if bsc_pdu is None: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "create", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + generate_keypair = "yes")) + elif bsc_pdu.signing_cert != bsc_cert or bsc_pdu.signing_cert_crl != bsc_crl: + rpkid_query.append(rpki.left_right.bsc_elt.make_pdu( + action = "set", + tag = "bsc", + self_handle = handle, + bsc_handle = bsc_handle, + signing_cert = bsc_cert, + signing_cert_crl = bsc_crl)) + + rpkid_query.extend(rpki.left_right.bsc_elt.make_pdu( + action = "destroy", self_handle = handle, bsc_handle = b) for b in bsc_pdus) + + bsc_req = None + + if bsc_pdu and bsc_pdu.pkcs10_request: + bsc_req = bsc_pdu.pkcs10_request + + # At present we need one <repository/> per <parent/>, not because + # rpkid requires that, but because pubd does. pubd probably should + # be fixed to support a single client allowed to update multiple + # trees, but for the moment the easiest way forward is just to + # enforce a 1:1 mapping between <parent/> and <repository/> objects + + for repository in tree.getiterator("repository"): + + repository_handle = repository.get("handle") + repository_pdu = repository_pdus.pop(repository_handle, None) + repository_uri = repository.get("service_uri") + repository_cert = findbase64(repository, "bpki_certificate") + + if (repository_pdu is None or + repository_pdu.bsc_handle != bsc_handle or + repository_pdu.peer_contact_uri != repository_uri or + repository_pdu.bpki_cert != repository_cert): + rpkid_query.append(rpki.left_right.repository_elt.make_pdu( + action = "create" if repository_pdu is None else "set", + tag = repository_handle, + self_handle = handle, + repository_handle = repository_handle, + bsc_handle = bsc_handle, + peer_contact_uri = repository_uri, + bpki_cert = repository_cert)) + + rpkid_query.extend(rpki.left_right.repository_elt.make_pdu( + action = "destroy", self_handle = handle, repository_handle = r) for r in repository_pdus) + + # <parent/> setup code currently assumes 1:1 mapping between + # <repository/> and <parent/>, and further assumes that the handles + # for an associated pair are the identical (that is: + # parent.repository_handle == parent.parent_handle). + + for parent in tree.getiterator("parent"): + + parent_handle = parent.get("handle") + parent_pdu = parent_pdus.pop(parent_handle, None) + parent_uri = parent.get("service_uri") + parent_myhandle = parent.get("myhandle") + parent_sia_base = parent.get("sia_base") + parent_cms_cert = findbase64(parent, "bpki_cms_certificate") + parent_https_cert = findbase64(parent, "bpki_https_certificate") + + if (parent_pdu is None or + parent_pdu.bsc_handle != bsc_handle or + parent_pdu.repository_handle != parent_handle or + parent_pdu.peer_contact_uri != parent_uri or + parent_pdu.sia_base != parent_sia_base or + parent_pdu.sender_name != parent_myhandle or + parent_pdu.recipient_name != parent_handle or + parent_pdu.bpki_cms_cert != parent_cms_cert or + parent_pdu.bpki_https_cert != parent_https_cert): + rpkid_query.append(rpki.left_right.parent_elt.make_pdu( + action = "create" if parent_pdu is None else "set", + tag = parent_handle, + self_handle = handle, + parent_handle = parent_handle, + bsc_handle = bsc_handle, + repository_handle = parent_handle, + peer_contact_uri = parent_uri, + sia_base = parent_sia_base, + sender_name = parent_myhandle, + recipient_name = parent_handle, + bpki_cms_cert = parent_cms_cert, + bpki_https_cert = parent_https_cert)) + + rpkid_query.extend(rpki.left_right.parent_elt.make_pdu( + action = "destroy", self_handle = handle, parent_handle = p) for p in parent_pdus) + + # Children are simpler than parents, because they call us, so no URL + # to construct and figuring out what certificate to use is their + # problem, not ours. + + for child in tree.getiterator("child"): + + child_handle = child.get("handle") + child_pdu = child_pdus.pop(child_handle, None) + child_cert = findbase64(child, "bpki_certificate") + + if (child_pdu is None or + child_pdu.bsc_handle != bsc_handle or + child_pdu.bpki_cert != child_cert): + rpkid_query.append(rpki.left_right.child_elt.make_pdu( + action = "create" if child_pdu is None else "set", + tag = child_handle, + self_handle = handle, + child_handle = child_handle, + bsc_handle = bsc_handle, + bpki_cert = child_cert)) + + rpkid_query.extend(rpki.left_right.child_elt.make_pdu( + action = "destroy", self_handle = handle, child_handle = c) for c in child_pdus) + + # Publication setup. + + if self.run_pubd: + + for f in self.entitydb.iterate("pubclients", "*.xml"): + c = etree_read(f) + + client_handle = c.get("client_handle") + client_base_uri = c.get("sia_base") + client_bpki_cert = rpki.x509.X509(PEM_file = self.bpki_servers.fxcert(c.findtext("bpki_client_ta"))) + client_pdu = client_pdus.pop(client_handle, None) + + if (client_pdu is None or + client_pdu.base_uri != client_base_uri or + client_pdu.bpki_cert != client_bpki_cert): + pubd_query.append(rpki.publication.client_elt.make_pdu( + action = "create" if client_pdu is None else "set", + client_handle = client_handle, + bpki_cert = client_bpki_cert, + base_uri = client_base_uri)) + + pubd_query.extend(rpki.publication.client_elt.make_pdu( + action = "destroy", client_handle = p) for p in client_pdus) + + # If we changed anything, ship updates off to daemons + + failed = False + + if rpkid_query: + rpkid_reply = call_rpkid(*rpkid_query) + bsc_pdus = dict((x.bsc_handle, x) for x in rpkid_reply if isinstance(x, rpki.left_right.bsc_elt)) + if bsc_handle in bsc_pdus and bsc_pdus[bsc_handle].pkcs10_request: + bsc_req = bsc_pdus[bsc_handle].pkcs10_request + for r in rpkid_reply: + if isinstance(r, rpki.left_right.report_error_elt): + failed = True + print "rpkid reported failure:", r.error_code + if r.error_text: + print r.error_text + + if failed: + raise RuntimeError + + if pubd_query: + assert self.run_pubd + pubd_reply = call_pubd(*pubd_query) + for r in pubd_reply: + if isinstance(r, rpki.publication.report_error_elt): + failed = True + print "pubd reported failure:", r.error_code + if r.error_text: + print r.error_text + + if failed: + raise RuntimeError + + # Rewrite XML. + + e = tree.find("bpki_bsc_pkcs10") + if e is not None: + tree.remove(e) + if bsc_req is not None: + SubElement(tree, "bpki_bsc_pkcs10").text = bsc_req.get_Base64() + + tree.set("service_uri", rpkid_base + "up-down/" + handle) + + e = tree.find("bpki_server_ta") + if e is not None: + tree.remove(e) + PEMElement(tree, "bpki_server_ta", self.bpki_resources.cer) + + etree_write(tree, xmlfile, validate = True, + msg = None if xmlfile is my_xmlfile else 'Send this file back to the hosted entity ("%s")' % handle) + + db.close() + + # Run event loop again to give TLS connections a chance to shut down cleanly. + # Might need to add a timeout here, dunno yet. + + rpki.async.event_loop() + + + +if __name__ == "__main__": + main() diff --git a/myrpki/myrpki.rnc b/myrpki/myrpki.rnc new file mode 100644 index 00000000..f1cfe249 --- /dev/null +++ b/myrpki/myrpki.rnc @@ -0,0 +1,135 @@ +# $Id$ +# +# RelaxNG Schema for MyRPKI XML messages. +# +# libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so +# run the compact syntax through trang to get XML syntax. +# +# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +default namespace = "http://www.hactrn.net/uris/rpki/myrpki/" + +version = "2" + +base64 = xsd:base64Binary { maxLength="512000" } +object_handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9]*" } +pubd_handle = xsd:string { maxLength="255" pattern="[\-_A-Za-z0-9/]*" } +uri = xsd:anyURI { maxLength="4096" } +asn = xsd:positiveInteger +asn_list = xsd:string { maxLength="512000" pattern="[\-,0-9]*" } +ipv4_list = xsd:string { maxLength="512000" pattern="[\-,0-9/.]*" } +ipv6_list = xsd:string { maxLength="512000" pattern="[\-,0-9/:a-fA-F]*" } +timestamp = xsd:dateTime { pattern=".*Z" } + +start |= element myrpki { + attribute version { version }, + attribute handle { object_handle }, + attribute service_uri { uri }?, + element roa_request { + attribute asn { asn }, + attribute v4 { ipv4_list }, + attribute v6 { ipv6_list } + }*, + element child { + attribute handle { object_handle }, + attribute valid_until { timestamp }, + attribute asns { asn_list }?, + attribute v4 { ipv4_list }?, + attribute v6 { ipv6_list }?, + element bpki_certificate { base64 }? + }*, + element parent { + attribute handle { object_handle }, + attribute service_uri { uri }?, + attribute myhandle { object_handle }?, + attribute sia_base { uri }?, + element bpki_cms_certificate { base64 }?, + element bpki_https_certificate { base64 }? + }*, + element repository { + attribute handle { object_handle }, + attribute service_uri { uri }?, + element bpki_certificate { base64 }? + }*, + element bpki_ca_certificate { base64 }?, + element bpki_crl { base64 }?, + element bpki_bsc_certificate { base64 }?, + element bpki_bsc_pkcs10 { base64 }?, + element bpki_server_ta { base64 }? +} + +start |= element identity { + attribute version { version }, + attribute handle { object_handle }, + element bpki_ta { base64 } +} + +authorization = element authorization { + attribute referrer { pubd_handle }, + base64 +} + +contact_info = element contact_info { + attribute uri { uri }?, + xsd:string +} + +repository_payload = ( + (attribute type { "offer" }) | + (attribute type { "referral" }, authorization, contact_info) +) + +start |= element parent { + attribute version { version }, + attribute valid_until { timestamp }, + attribute service_uri { uri }?, + attribute child_handle { object_handle }, + attribute parent_handle { object_handle }, + element bpki_resource_ta { base64 }, + element bpki_server_ta { base64 }, + element bpki_child_ta { base64 }, + element repository { repository_payload }? +} + +start |= element repository { + attribute version { version }, + attribute handle { object_handle }, + attribute parent_handle { object_handle }, + repository_payload, + element bpki_client_ta { base64 } +} + +start |= element repository { + attribute version { version }, + attribute type { "confirmed" }, + attribute parent_handle { object_handle }, + attribute client_handle { pubd_handle }, + attribute service_uri { uri }, + attribute sia_base { uri }, + element bpki_server_ta { base64 }, + element bpki_client_ta { base64 }, + authorization?, + contact_info? +} + +start |= element referral { + attribute version { version }, + attribute authorized_sia_base { uri }, + base64 +} + +# Local Variables: +# indent-tabs-mode: nil +# End: diff --git a/myrpki/myrpki.rng b/myrpki/myrpki.rng new file mode 100644 index 00000000..dc4f18e6 --- /dev/null +++ b/myrpki/myrpki.rng @@ -0,0 +1,355 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + $Id: myrpki.rnc 3105 2010-03-16 22:24:19Z sra $ + + RelaxNG Schema for MyRPKI XML messages. + + libxml2 (including xmllint) only groks the XML syntax of RelaxNG, so + run the compact syntax through trang to get XML syntax. + + Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE + OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. +--> +<grammar ns="http://www.hactrn.net/uris/rpki/myrpki/" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes"> + <define name="version"> + <value>2</value> + </define> + <define name="base64"> + <data type="base64Binary"> + <param name="maxLength">512000</param> + </data> + </define> + <define name="object_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9]*</param> + </data> + </define> + <define name="pubd_handle"> + <data type="string"> + <param name="maxLength">255</param> + <param name="pattern">[\-_A-Za-z0-9/]*</param> + </data> + </define> + <define name="uri"> + <data type="anyURI"> + <param name="maxLength">4096</param> + </data> + </define> + <define name="asn"> + <data type="positiveInteger"/> + </define> + <define name="asn_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9]*</param> + </data> + </define> + <define name="ipv4_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/.]*</param> + </data> + </define> + <define name="ipv6_list"> + <data type="string"> + <param name="maxLength">512000</param> + <param name="pattern">[\-,0-9/:a-fA-F]*</param> + </data> + </define> + <define name="timestamp"> + <data type="dateTime"> + <param name="pattern">.*Z</param> + </data> + </define> + <start combine="choice"> + <element name="myrpki"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <zeroOrMore> + <element name="roa_request"> + <attribute name="asn"> + <ref name="asn"/> + </attribute> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="child"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="valid_until"> + <ref name="timestamp"/> + </attribute> + <optional> + <attribute name="asns"> + <ref name="asn_list"/> + </attribute> + </optional> + <optional> + <attribute name="v4"> + <ref name="ipv4_list"/> + </attribute> + </optional> + <optional> + <attribute name="v6"> + <ref name="ipv6_list"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="parent"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <attribute name="myhandle"> + <ref name="object_handle"/> + </attribute> + </optional> + <optional> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_cms_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_https_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <zeroOrMore> + <element name="repository"> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <optional> + <element name="bpki_certificate"> + <ref name="base64"/> + </element> + </optional> + </element> + </zeroOrMore> + <optional> + <element name="bpki_ca_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_crl"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_bsc_certificate"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_bsc_pkcs10"> + <ref name="base64"/> + </element> + </optional> + <optional> + <element name="bpki_server_ta"> + <ref name="base64"/> + </element> + </optional> + </element> + </start> + <start combine="choice"> + <element name="identity"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <element name="bpki_ta"> + <ref name="base64"/> + </element> + </element> + </start> + <define name="authorization"> + <element name="authorization"> + <attribute name="referrer"> + <ref name="pubd_handle"/> + </attribute> + <ref name="base64"/> + </element> + </define> + <define name="contact_info"> + <element name="contact_info"> + <optional> + <attribute name="uri"> + <ref name="uri"/> + </attribute> + </optional> + <data type="string"/> + </element> + </define> + <define name="repository_payload"> + <choice> + <attribute name="type"> + <value>offer</value> + </attribute> + <group> + <attribute name="type"> + <value>referral</value> + </attribute> + <ref name="authorization"/> + <ref name="contact_info"/> + </group> + </choice> + </define> + <start combine="choice"> + <element name="parent"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="valid_until"> + <ref name="timestamp"/> + </attribute> + <optional> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + </optional> + <attribute name="child_handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <element name="bpki_resource_ta"> + <ref name="base64"/> + </element> + <element name="bpki_server_ta"> + <ref name="base64"/> + </element> + <element name="bpki_child_ta"> + <ref name="base64"/> + </element> + <optional> + <element name="repository"> + <ref name="repository_payload"/> + </element> + </optional> + </element> + </start> + <start combine="choice"> + <element name="repository"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <ref name="repository_payload"/> + <element name="bpki_client_ta"> + <ref name="base64"/> + </element> + </element> + </start> + <start combine="choice"> + <element name="repository"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="type"> + <value>confirmed</value> + </attribute> + <attribute name="parent_handle"> + <ref name="object_handle"/> + </attribute> + <attribute name="client_handle"> + <ref name="pubd_handle"/> + </attribute> + <attribute name="service_uri"> + <ref name="uri"/> + </attribute> + <attribute name="sia_base"> + <ref name="uri"/> + </attribute> + <element name="bpki_server_ta"> + <ref name="base64"/> + </element> + <element name="bpki_client_ta"> + <ref name="base64"/> + </element> + <optional> + <ref name="authorization"/> + </optional> + <optional> + <ref name="contact_info"/> + </optional> + </element> + </start> + <start combine="choice"> + <element name="referral"> + <attribute name="version"> + <ref name="version"/> + </attribute> + <attribute name="authorized_sia_base"> + <ref name="uri"/> + </attribute> + <ref name="base64"/> + </element> + </start> +</grammar> +<!-- + Local Variables: + indent-tabs-mode: nil + End: +--> diff --git a/myrpki/rcynic.conf b/myrpki/rcynic.conf new file mode 100644 index 00000000..02a2495b --- /dev/null +++ b/myrpki/rcynic.conf @@ -0,0 +1,11 @@ +# $Id$ + +[rcynic] +xml-summary = rcynic.xml +jitter = 0 +use-links = yes +use-syslog = no +use-stderr = yes +log-level = log_debug + +trust-anchor = test/RIR/publication/root.cer diff --git a/myrpki/ripe-asns-to-csv.py b/myrpki/ripe-asns-to-csv.py new file mode 100644 index 00000000..04a92627 --- /dev/null +++ b/myrpki/ripe-asns-to-csv.py @@ -0,0 +1,106 @@ +""" +Parse a WHOIS research dump and write out (just) the RPKI-relevant +fields in myrpki-format CSV syntax. + +NB: The input data for this script is publicly available via FTP, but +you'll have to fetch the data from RIPE yourself, and be sure to see +the terms and conditions referenced by the data file header comments. + +$Id$ + +Copyright (C) 2009 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import gzip, csv, myrpki + +class Handle(dict): + + want_tags = () + + debug = False + + def set(self, tag, val): + if tag in self.want_tags: + self[tag] = "".join(val.split(" ")) + + def check(self): + for tag in self.want_tags: + if not tag in self: + return False + if self.debug: + self.log() + return True + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, + " ".join("%s:%s" % (tag, self.get(tag, "?")) + for tag in self.want_tags)) + + def log(self): + print repr(self) + + def finish(self, ctx): + self.check() + +class aut_num(Handle): + want_tags = ("aut-num", "mnt-by", "as-name") + + def set(self, tag, val): + if tag == "aut-num" and val.startswith("AS"): + val = val[2:] + Handle.set(self, tag, val) + + def finish(self, ctx): + if self.check(): + ctx.asns.writerow((self["mnt-by"], self["aut-num"])) + +class main(object): + + types = dict((x.want_tags[0], x) for x in (aut_num,)) + + + def finish_statement(self, done): + if self.statement: + tag, sep, val = self.statement.partition(":") + assert sep, "Couldn't find separator in %r" % self.statement + tag = tag.strip().lower() + val = val.strip().upper() + if self.cur is None: + self.cur = self.types[tag]() if tag in self.types else False + if self.cur is not False: + self.cur.set(tag, val) + if done and self.cur: + self.cur.finish(self) + self.cur = None + + filenames = ("ripe.db.aut-num.gz",) + + def __init__(self): + self.asns = myrpki.csv_writer("asns.csv") + for fn in self.filenames: + f = gzip.open(fn) + self.statement = "" + self.cur = None + for line in f: + line = line.expandtabs().partition("#")[0].rstrip("\n") + if line and not line[0].isalpha(): + self.statement += line[1:] if line[0] == "+" else line + else: + self.finish_statement(not line) + self.statement = line + self.finish_statement(True) + f.close() + +main() diff --git a/myrpki/ripe-prefixes-to-csv.awk b/myrpki/ripe-prefixes-to-csv.awk new file mode 100644 index 00000000..582d5ce7 --- /dev/null +++ b/myrpki/ripe-prefixes-to-csv.awk @@ -0,0 +1,43 @@ +#!/usr/bin/awk -f +# $Id$ + +# ftp -pa ftp://ftp.ripe.net/pub/stats/ripencc/membership/alloclist.txt + +BEGIN { + translation["ie.google"] = "GoogleIreland"; +} + +function done() { + if (handle in translation) + handle = translation[handle]; + for (i = 1; i <= n_allocs; i++) + print handle "\t" alloc[i]; + n_allocs = 0; +} + +/^[a-z]/ { + done(); + handle = $0; + nr = NR; +} + +NR == nr + 1 { + name = $0; +} + +NR > nr + 2 && NF > 1 && $2 !~ /:/ { + split($2, a, "/"); + len = a[2]; + split(a[1], a, /[.]/); + for (i = length(a); i < 4; i++) + a[i+1] = 0; + alloc[++n_allocs] = sprintf("%d.%d.%d.%d/%d", a[1], a[2], a[3], a[4], len); +} + +NR > nr + 2 && NF > 1 && $2 ~ /:/ { + alloc[++n_allocs] = $2; +} + +END { + done(); +} diff --git a/myrpki/rpki b/myrpki/rpki new file mode 120000 index 00000000..168548eb --- /dev/null +++ b/myrpki/rpki @@ -0,0 +1 @@ +../rpkid/rpki
\ No newline at end of file diff --git a/myrpki/setup-rootd.sh b/myrpki/setup-rootd.sh new file mode 100644 index 00000000..001ed862 --- /dev/null +++ b/myrpki/setup-rootd.sh @@ -0,0 +1,36 @@ +#!/bin/sh - +# +# $Id$ +# +# Copyright (C) 2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +# Setting up rootd requires cross-certifying rpkid's resource-holding +# BPKI trust anchor under the BPKI trust anchor that rootd uses. This +# script handles that, albiet in a very ugly way. +# +# Filenames are wired in, you might need to change these if you've +# done something more complicated. + +export RANDFILE=.OpenSSL.whines.unless.I.set.this +export BPKI_DIRECTORY=`pwd`/bpki/servers + +openssl=../openssl/openssl/apps/openssl + +$openssl ca -notext -batch -config myrpki.conf \ + -ss_cert bpki/resources/ca.cer \ + -out $BPKI_DIRECTORY/child.cer \ + -extensions ca_x509_ext_xcert0 + +$openssl x509 -noout -text -in $BPKI_DIRECTORY/child.cer diff --git a/myrpki/sql-cleaner.py b/myrpki/sql-cleaner.py new file mode 100644 index 00000000..d7e1a568 --- /dev/null +++ b/myrpki/sql-cleaner.py @@ -0,0 +1,35 @@ +""" +(Re)Initialize SQL tables used by these programs. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, rpki.config + +cfg = rpki.config.parser("yamltest.conf", "yamltest", allow_missing = True) + +for name in ("rpkid", "irdbd", "pubd"): + + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") + + databases = [name[:4]] + databases.extend("%s%d" % (name[:4], i) for i in xrange(12)) + + for db in databases: + subprocess.check_call(("mysql", "-u", username, "-p" + password, db), + stdin = open("../rpkid/%s.sql" % name)) diff --git a/myrpki/sql-dumper.py b/myrpki/sql-dumper.py new file mode 100644 index 00000000..4437d858 --- /dev/null +++ b/myrpki/sql-dumper.py @@ -0,0 +1,32 @@ +""" +Dump backup copies of SQL tables used by these programs. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, rpki.config + +cfg = rpki.config.parser("yamltest.conf", "yamltest") + +for name in ("rpkid", "irdbd", "pubd"): + + username = cfg.get("%s_sql_username" % name, name[:4]) + password = cfg.get("%s_sql_password" % name, "fnord") + + cmd = ["mysqldump", "-u", username, "-p" + password, "--databases", name[:4]] + cmd.extend("%s%d" % (name[:4], i) for i in xrange(12)) + subprocess.check_call(cmd, stdout = open("backup.%s.sql" % name, "w")) diff --git a/myrpki/sql-setup.py b/myrpki/sql-setup.py new file mode 100644 index 00000000..78907321 --- /dev/null +++ b/myrpki/sql-setup.py @@ -0,0 +1,107 @@ +""" +Automated setup of all the pesky SQL stuff we need. Prompts for MySQL +root password, pulls other information from myrpki.conf. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +from __future__ import with_statement + +import os, getopt, sys, time, rpki.config, getpass, warnings + +# Silence warning while loading MySQLdb in Python 2.6, sigh +if hasattr(warnings, "catch_warnings"): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + import MySQLdb +else: + import MySQLdb + +import _mysql_exceptions + +warnings.simplefilter("error", _mysql_exceptions.Warning) + +schema_dir = os.path.normpath(os.path.join(sys.path[0], "../rpkid")) + +def read_schema(filename): + """ + Convert an SQL file into a list of SQL statements. + """ + lines = [] + f = open(filename) + for line in f: + line = " ".join(line.split()) + if line and not line.startswith("--"): + lines.append(line) + f.close() + return [statement.strip() for statement in " ".join(lines).rstrip(";").split(";")] + +def sql_setup(name): + """ + Create a new SQL database and construct all its tables. + """ + database = cfg.get("sql-database", section = name) + username = cfg.get("sql-username", section = name) + password = cfg.get("sql-password", section = name) + schema = read_schema(os.path.join(schema_dir, "%s.sql" % name)) + + print "Creating database", database + cur = rootdb.cursor() + try: + cur.execute("DROP DATABASE IF EXISTS %s" % database) + except: + pass + cur.execute("CREATE DATABASE %s" % database) + cur.execute("GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY %%s" % (database, username), (password,)) + rootdb.commit() + + db = MySQLdb.connect(db = database, user = username, passwd = password) + cur = db.cursor() + for statement in schema: + if statement.upper().startswith("DROP TABLE"): + continue + if verbose: + print "+", statement + cur.execute(statement) + db.commit() + db.close() + +cfg_file = "myrpki.conf" + +verbose = False + +opts, argv = getopt.getopt(sys.argv[1:], "c:hv?", ["config=", "help", "verbose"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-v", "--verbose"): + verbose = True + if o in ("-c", "--config"): + cfg_file = a + +cfg = rpki.config.parser(cfg_file, "myrpki") + +rootdb = MySQLdb.connect(db = "mysql", user = "root", passwd = getpass.getpass("Please enter your MySQL root password: ")) + +sql_setup("irdbd") +sql_setup("rpkid") + +if cfg.getboolean("run_pubd", False): + sql_setup("pubd") + +rootdb.close() diff --git a/myrpki/start-servers.py b/myrpki/start-servers.py new file mode 100644 index 00000000..da958812 --- /dev/null +++ b/myrpki/start-servers.py @@ -0,0 +1,73 @@ +""" +Start servers, logging to files, looking at config file to figure out +which servers the user wants started. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +""" + +import subprocess, os, getopt, sys, time, rpki.config + +rpkid_dir = os.path.normpath(os.path.join(sys.path[0], "../rpkid")) + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "myrpki.conf" +debug = False + +opts, argv = getopt.getopt(sys.argv[1:], "c:dh?", ["config=", "debug" "help"]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + elif o in ("-c", "--config"): + cfg_file = a + elif o in ("-d", "--debug"): + debug = True + +names = ["irdbd", "rpkid"] + +cfg = rpki.config.parser(cfg_file, "myrpki") + +if cfg.getboolean("run_pubd", False): + names.append("pubd") + +if cfg.getboolean("run_rootd", False): + names.append("rootd") + +for name in names: + cmd = ("python", os.path.join(rpkid_dir, name + ".py"), "-c", cfg_file) + if debug: + proc = subprocess.Popen(cmd + ("-d",), stdout = open(name + ".log", "a"), stderr = subprocess.STDOUT) + else: + proc = subprocess.Popen(cmd) + print ("Started %r, pid %s" if proc.poll() is None else "Problem starting %r, pid %s") % (name, proc.pid) diff --git a/myrpki/test-all.sh b/myrpki/test-all.sh new file mode 100644 index 00000000..35026f7e --- /dev/null +++ b/myrpki/test-all.sh @@ -0,0 +1,45 @@ +#!/bin/sh - +# $Id$ + +# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +set -x + +export TZ=UTC MYRPKI_RNG=`pwd`/myrpki.rng + +test -z "$STY" && exec screen -L sh $0 + +screen -X split +screen -X focus + +for i in ../rpkid/testbed.*.yaml +do + rm -rf test + python sql-cleaner.py + screen python yamltest.py -p yamltest.pid $i + date + sleep 180 + for j in . . . . . . . . . . + do + sleep 30 + date + ../rcynic/rcynic + ../rcynic/show.sh + date + done + test -r yamltest.pid && kill -INT `cat yamltest.pid` + sleep 30 + make backup +done diff --git a/myrpki/test-myrpki-cms.py b/myrpki/test-myrpki-cms.py new file mode 100644 index 00000000..29bea39c --- /dev/null +++ b/myrpki/test-myrpki-cms.py @@ -0,0 +1,66 @@ +""" +Scratch pad for working out what CMS referral code looks like. + +This is only in subversion for archival and backup, I don't expect +users to run this, and will delete it in the near future. + + +$Id$ + +Copyright (C) 2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import subprocess, os, sys, myrpki + +original_xml = '''\ +<publication_referral xmlns="http://www.hactrn.net/uris/rpki/publication-spec/" + sia_base=rsync://repository.example/path/to/me/space-i-give-to-my-child"> + Base64 encoded BPKI TA of resource holding aspect of my child xxx blah blah blah blah xxx +</publication_referral> +''' + +f = open("original.xml", "w") +f.write(original_xml) +f.close() + +myrpki.openssl = "/u/sra/rpki/subvert-rpki.hactrn.net/openssl/openssl/apps/openssl" +os.putenv("OPENSSL_CONF", "/dev/null") + +bpki = myrpki.CA("test/Alice/myrpki.conf", "test/Alice/bpki/resources") +bpki.ee("/CN=Alice Signed Referral CMS Test EE Certificate", "CMSEE") + +# "id-ct-xml" from rpki.oids +oid = ".".join(map(str, (1, 2, 840, 113549, 1, 9, 16, 1, 28))) + +format = "DER" # PEM or DER + +subprocess.check_call((myrpki.openssl, "cms", "-sign", + "-binary", "-nodetach", "-nosmimecap", "-keyid", "-outform", format, + "-econtent_type", oid, "-md", "sha256", + "-inkey", "test/Alice/bpki/resources/CMSEE.key", + "-signer", "test/Alice/bpki/resources/CMSEE.cer", + "-in", "original.xml", + "-out", "original.%s" % format.lower())) + +if format == "DER": + subprocess.call(("dumpasn1", "-a", "original.cms")) + +# verifying may not be necessary here, that might be pubd's job. or +# at least we can make it the job of the code formerly known as irdbd, +# where we have full libraries available to us. but blunder ahead... + +subprocess.check_call((myrpki.openssl, "cms", "-verify", "-inform", format, + "-CAfile", "test/Alice/bpki/resources/ca.cer", + "-in", "original.%s" % format.lower())) diff --git a/myrpki/testbed-rootcert.py b/myrpki/testbed-rootcert.py new file mode 100644 index 00000000..54d1480c --- /dev/null +++ b/myrpki/testbed-rootcert.py @@ -0,0 +1,65 @@ +""" +Generate config for a test RPKI root certificate for resources +specified in asns.csv and prefixes.csv. + +This script is separate from arin-to-csv.py so that we can convert on +the fly rather than having to pull the entire database into memory. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import csv, myrpki, sys + +if len(sys.argv) != 2: + raise RuntimeError, "Usage: %s [holder]" % sys.argv[0] + +print '''\ +[req] +default_bits = 2048 +default_md = sha256 +distinguished_name = req_dn +prompt = no +encrypt_key = no + +[req_dn] +CN = Pseudo-%(HOLDER)s testbed root RPKI certificate + +[x509v3_extensions] +basicConstraints = critical,CA:true +subjectKeyIdentifier = hash +keyUsage = critical,keyCertSign,cRLSign +subjectInfoAccess = 1.3.6.1.5.5.7.48.5;URI:rsync://%(holder)s.rpki.net/rpki/%(holder)s/,1.3.6.1.5.5.7.48.10;URI:rsync://%(holder)s.rpki.net/rpki/%(holder)s/root.mnf +certificatePolicies = critical,1.3.6.1.5.5.7.14.2 +sbgp-autonomousSysNum = critical,@rfc3779_asns +sbgp-ipAddrBlock = critical,@rfc3997_addrs + +[rfc3779_asns] +''' % { "holder" : sys.argv[1].lower(), + "HOLDER" : sys.argv[1].upper() } + +for i, asn in enumerate(asn for handle, asn in myrpki.csv_reader("asns.csv", columns = 2)): + print "AS.%d = %s" % (i, asn) + +print '''\ + +[rfc3997_addrs] + +''' + +for i, prefix in enumerate(prefix for handle, prefix in myrpki.csv_reader("prefixes.csv", columns = 2)): + v = 6 if ":" in prefix else 4 + print "IPv%d.%d = %s" % (v, i, prefix) diff --git a/myrpki/translate-handles.py b/myrpki/translate-handles.py new file mode 100644 index 00000000..308b878e --- /dev/null +++ b/myrpki/translate-handles.py @@ -0,0 +1,49 @@ +""" +Translate handles from the ones provided in a database dump into the +ones we use in our testbed. This has been broken out into a separate +program for two reasons: + +- Conversion of some of the RIR data is a very slow process, and it's + both annoying and unnecessary to run it every time we add a new + participant to the testbed. + +- This handle translation business now has fingers into half a dozen + scripts, so it needs refactoring in any case, either as a common + library function or as a separate script. + +This program takes a list of .CSV files on its command line, and +rewrites them as needed after performing the translation. + +$Id$ + +Copyright (C) 2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import os, sys, myrpki + +translations = dict((src, dst) for src, dst in myrpki.csv_reader("translations.csv", columns = 2)) + +for filename in sys.argv[1:]: + + tmpfile = "%s.%d" % os.getpid() + csvout = myrpki.csv_writer(tmpfile) + + for cols in myrpki.csv_reader(filename): + if cols[0] in translations: + cols[0] = translations[cols[0]] + csvout(cols) + + del csvout + os.rename(tmpfile, filename) diff --git a/myrpki/verify-bpki.sh b/myrpki/verify-bpki.sh new file mode 100755 index 00000000..0e36d796 --- /dev/null +++ b/myrpki/verify-bpki.sh @@ -0,0 +1,43 @@ +#!/bin/sh - +# $Id$ +# +# Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +# PERFORMANCE OF THIS SOFTWARE. + +# Tests of generated BPKI certificates. Kind of cheesy, but does test +# the basic stuff. + +exec 2>&1 + +for bpki in bpki/* +do + crls=$(find $bpki -name '*.crl') + + # Check that CRLs verify properly + for crl in $crls + do + echo -n "$crl: " + openssl crl -CAfile $bpki/ca.cer -noout -in $crl + done + + # Check that issued certificates verify properly + cat $bpki/ca.cer $crls | openssl verify -crl_check -CAfile /dev/stdin $(find $bpki -name '*.cer' ! -name 'ca.cer' ! -name '*.cacert.cer') + +done + +# Check that cross-certified BSC certificates verify properly +if test -d bpki/servers +then + cat bpki/servers/xcert.*.cer | openssl verify -verbose -CAfile bpki/servers/ca.cer -untrusted /dev/stdin bpki/resources/bsc.*.cer +fi diff --git a/myrpki/xml-parse-test.py b/myrpki/xml-parse-test.py new file mode 100644 index 00000000..17b1884b --- /dev/null +++ b/myrpki/xml-parse-test.py @@ -0,0 +1,101 @@ +""" +Test parser and display tool for myrpki.xml files. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. +""" + +import lxml.etree, rpki.resource_set, base64, subprocess + +relaxng = lxml.etree.RelaxNG(file = "myrpki.rng") + +tree = lxml.etree.parse("myrpki.xml").getroot() + +if False: + print lxml.etree.tostring(tree, pretty_print = True, encoding = "us-ascii", xml_declaration = True) + +relaxng.assertValid(tree) + +def showitems(x): + if False: + for k, v in x.items(): + if v: + print " ", k, v + +def tag(t): + return "{http://www.hactrn.net/uris/rpki/myrpki/}" + t + +print "My handle:", tree.get("handle") + +print "Children:" +for x in tree.getiterator(tag("child")): + print " ", x + print " Handle:", x.get("handle") + print " ASNS: ", rpki.resource_set.resource_set_as(x.get("asns")) + print " IPv4: ", rpki.resource_set.resource_set_ipv4(x.get("v4")) + print " Valid: ", x.get("valid_until") + showitems(x) +print + +print "ROA requests:" +for x in tree.getiterator(tag("roa_request")): + print " ", x + print " ASN: ", x.get("asn") + print " IPv4:", rpki.resource_set.roa_prefix_set_ipv4(x.get("v4")) + print " IPv6:", rpki.resource_set.roa_prefix_set_ipv6(x.get("v6")) + showitems(x) +print + +def showpem(label, b64, kind): + cmd = ("openssl", kind, "-noout", "-text", "-inform", "DER") + if kind == "x509": + cmd += ("-certopt", "no_pubkey,no_sigdump") + p = subprocess.Popen(cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE) + text = p.communicate(input = base64.b64decode(b64))[0] + if p.returncode != 0: + raise subprocess.CalledProcessError(returncode = p.returncode, cmd = cmd) + print label, text + +for x in tree.getiterator(tag("child")): + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Child", cert, "x509") + +for x in tree.getiterator(tag("parent")): + print "Parent URI:", x.get("service_uri") + cert = x.findtext(tag("bpki_certificate")) + if cert: + showpem("Parent", cert, "x509") + +ca = tree.findtext(tag("bpki_ca_certificate")) +if ca: + showpem("CA", ca, "x509") + +bsc = tree.findtext(tag("bpki_bsc_certificate")) +if bsc: + showpem("BSC EE", bsc, "x509") + +repo = tree.findtext(tag("bpki_repository_certificate")) +if repo: + showpem("Repository", repo, "x509") + +req = tree.findtext(tag("bpki_bsc_pkcs10")) +if req: + showpem("BSC EE", req, "req") + +crl = tree.findtext(tag("bpki_crl")) +if crl: + showpem("CA", crl, "crl") diff --git a/myrpki/yamltest.py b/myrpki/yamltest.py new file mode 100644 index 00000000..08153209 --- /dev/null +++ b/myrpki/yamltest.py @@ -0,0 +1,704 @@ +""" +Test framework, using the same YAML test description format as +testbed.py, but using the myrpki.py and myirbe.py tools to do all the +back-end work. Reads YAML file, generates .csv and .conf files, runs +daemons and waits for one of them to exit. + +Much of the YAML handling code lifted from testbed.py. + +Still to do: + +- Implement testebd.py-style delta actions, that is, modify the + allocation database under control of the YAML file, dump out new + .csv files, and run myrpki.py and myirbe.py again to feed resulting + changes into running daemons. + +$Id$ + +Copyright (C) 2009-2010 Internet Systems Consortium ("ISC") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +Portions copyright (C) 2007--2008 American Registry for Internet Numbers ("ARIN") + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND ARIN DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL ARIN BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +""" + +import subprocess, csv, re, os, getopt, sys, base64, yaml, signal, errno, time +import rpki.resource_set, rpki.sundial, rpki.config, rpki.log, myrpki + +# Nasty regular expressions for parsing config files. Sadly, while +# the Python ConfigParser supports writing config files, it does so in +# such a limited way that it's easier just to hack this ourselves. + +section_regexp = re.compile("\s*\[\s*(.+?)\s*\]\s*$") +variable_regexp = re.compile("\s*([-a-zA-Z0-9_]+)\s*=\s*(.+?)\s*$") + +def cleanpath(*names): + """ + Construct normalized pathnames. + """ + return os.path.normpath(os.path.join(*names)) + +# Pathnames for various things we need + +this_dir = os.getcwd() +test_dir = cleanpath(this_dir, "test") +rpkid_dir = cleanpath(this_dir, "../rpkid") + +prog_setup = cleanpath(this_dir, "myrpki.py") +prog_rpkid = cleanpath(rpkid_dir, "rpkid.py") +prog_irdbd = cleanpath(rpkid_dir, "irdbd.py") +prog_pubd = cleanpath(rpkid_dir, "pubd.py") +prog_rootd = cleanpath(rpkid_dir, "rootd.py") + +prog_openssl = cleanpath(this_dir, "../openssl/openssl/apps/openssl") + +class roa_request(object): + """ + Representation of a ROA request. + """ + + def __init__(self, asn, ipv4, ipv6): + self.asn = asn + self.v4 = rpki.resource_set.roa_prefix_set_ipv4("".join(ipv4.split())) if ipv4 else None + self.v6 = rpki.resource_set.roa_prefix_set_ipv6("".join(ipv6.split())) if ipv6 else None + + def __eq__(self, other): + return self.asn == other.asn and self.v4 == other.v4 and self.v6 == other.v6 + + def __hash__(self): + v4 = tuple(self.v4) if self.v4 is not None else None + v6 = tuple(self.v6) if self.v6 is not None else None + return self.asn.__hash__() + v4.__hash__() + v6.__hash__() + + def __str__(self): + if self.v4 and self.v6: + return "%s: %s,%s" % (self.asn, self.v4, self.v6) + else: + return "%s: %s" % (self.asn, self.v4 or self.v6) + + @classmethod + def parse(cls, yaml): + """ + Parse a ROA request from YAML format. + """ + return cls(yaml.get("asn"), yaml.get("ipv4"), yaml.get("ipv6")) + +class allocation_db(list): + """ + Our allocation database. + """ + + def __init__(self, yaml): + list.__init__(self) + self.root = allocation(yaml, self) + assert self.root.is_root() + if self.root.crl_interval is None: + self.root.crl_interval = 24 * 60 * 60 + if self.root.regen_margin is None: + self.root.regen_margin = 24 * 60 * 60 + for a in self: + if a.sia_base is None: + if a.runs_pubd(): + base = "rsync://localhost:%d/rpki/" % a.rsync_port + else: + base = a.parent.sia_base + a.sia_base = base + a.name + "/" + if a.base.valid_until is None: + a.base.valid_until = a.parent.base.valid_until + if a.crl_interval is None: + a.crl_interval = a.parent.crl_interval + if a.regen_margin is None: + a.regen_margin = a.parent.regen_margin + a.client_handle = "/".join(a.sia_base.rstrip("/").split("/")[3:]) + self.root.closure() + self.map = dict((a.name, a) for a in self) + for a in self: + if a.is_hosted(): + a.hosted_by = self.map[a.hosted_by] + a.hosted_by.hosts.append(a) + assert not a.is_root() and not a.hosted_by.is_hosted() + + def dump(self): + """ + Show contents of allocation database. + """ + for a in self: + a.dump() + + +class allocation(object): + """ + One entity in our allocation database. Every entity in the database + is assumed to hold resources, so needs at least myrpki services. + Entities that don't have the hosted_by property run their own copies + of rpkid, irdbd, and pubd, so they also need myirbe services. + """ + + base_port = 4400 + parent = None + crl_interval = None + regen_margin = None + rootd_port = None + engine = -1 + rpkid_port = -1 + irdbd_port = -1 + pubd_port = -1 + rsync_port = -1 + rootd_port = -1 + + @classmethod + def allocate_port(cls): + """ + Allocate a TCP port. + """ + cls.base_port += 1 + return cls.base_port + + base_engine = -1 + + @classmethod + def allocate_engine(cls): + """ + Allocate an engine number, mostly used to construct MySQL database + names. + """ + cls.base_engine += 1 + return cls.base_engine + + def __init__(self, yaml, db, parent = None): + db.append(self) + self.name = yaml["name"] + self.parent = parent + self.kids = [allocation(k, db, self) for k in yaml.get("kids", ())] + valid_until = None + if "valid_until" in yaml: + valid_until = rpki.sundial.datetime.fromdatetime(yaml.get("valid_until")) + if valid_until is None and "valid_for" in yaml: + valid_until = rpki.sundial.now() + rpki.sundial.timedelta.parse(yaml["valid_for"]) + self.base = rpki.resource_set.resource_bag( + asn = rpki.resource_set.resource_set_as(yaml.get("asn")), + v4 = rpki.resource_set.resource_set_ipv4(yaml.get("ipv4")), + v6 = rpki.resource_set.resource_set_ipv6(yaml.get("ipv6")), + valid_until = valid_until) + self.sia_base = yaml.get("sia_base") + if "crl_interval" in yaml: + self.crl_interval = rpki.sundial.timedelta.parse(yaml["crl_interval"]).convert_to_seconds() + if "regen_margin" in yaml: + self.regen_margin = rpki.sundial.timedelta.parse(yaml["regen_margin"]).convert_to_seconds() + self.roa_requests = [roa_request.parse(y) for y in yaml.get("roa_request", yaml.get("route_origin", ()))] + for r in self.roa_requests: + if r.v4: + self.base.v4 = self.base.v4.union(r.v4.to_resource_set()) + if r.v6: + self.base.v6 = self.base.v6.union(r.v6.to_resource_set()) + self.hosted_by = yaml.get("hosted_by") + self.hosts = [] + if not self.is_hosted(): + self.engine = self.allocate_engine() + self.rpkid_port = self.allocate_port() + self.irdbd_port = self.allocate_port() + if self.runs_pubd(): + self.pubd_port = self.allocate_port() + self.rsync_port = self.allocate_port() + if self.is_root(): + self.rootd_port = self.allocate_port() + + def closure(self): + """ + Compute resource closure of this node and its children, to avoid a + lot of tedious (and error-prone) duplication in the YAML file. + """ + resources = self.base + for kid in self.kids: + resources = resources.union(kid.closure()) + self.resources = resources + return resources + + def dump(self): + """ + Show content of this allocation node. + """ + print str(self) + + def __str__(self): + s = self.name + ":\n" + if self.resources.asn: s += " ASNs: %s\n" % self.resources.asn + if self.resources.v4: s += " IPv4: %s\n" % self.resources.v4 + if self.resources.v6: s += " IPv6: %s\n" % self.resources.v6 + if self.kids: s += " Kids: %s\n" % ", ".join(k.name for k in self.kids) + if self.parent: s += " Up: %s\n" % self.parent.name + if self.sia_base: s += " SIA: %s\n" % self.sia_base + if self.is_hosted(): s += " Host: %s\n" % self.hosted_by.name + if self.hosts: s += " Hosts: %s\n" % ", ".join(h.name for h in self.hosts) + for r in self.roa_requests: s += " ROA: %s\n" % r + if not self.is_hosted(): s += " IPort: %s\n" % self.irdbd_port + if self.runs_pubd(): s += " PPort: %s\n" % self.pubd_port + if not self.is_hosted(): s += " RPort: %s\n" % self.rpkid_port + if self.runs_pubd(): s += " SPort: %s\n" % self.rsync_port + if self.is_root(): s += " TPort: %s\n" % self.rootd_port + return s + " Until: %s\n" % self.resources.valid_until + + def is_root(self): + """ + Is this the root node? + """ + return self.parent is None + + def is_hosted(self): + """ + Is this entity hosted? + """ + return self.hosted_by is not None + + def runs_pubd(self): + """ + Does this entity run a pubd? + """ + return self.is_root() or not (self.is_hosted() or only_one_pubd) + + def path(self, *names): + """ + Construct pathnames in this entity's test directory. + """ + return cleanpath(test_dir, self.name, *names) + + def csvout(self, fn): + """ + Open and log a CSV output file. We use delimiter and dialect + settings imported from the myrpki module, so that we automatically + write CSV files in the right format. + """ + path = self.path(fn) + print "Writing", path + return myrpki.csv_writer(path) + + def up_down_url(self): + """ + Construct service URL for this node's parent. + """ + parent_port = self.parent.hosted_by.rpkid_port if self.parent.is_hosted() else self.parent.rpkid_port + return "https://localhost:%d/up-down/%s/%s" % (parent_port, self.parent.name, self.name) + + def dump_asns(self, fn): + """ + Write Autonomous System Numbers CSV file. + """ + f = self.csvout(fn) + for k in self.kids: + f.writerows((k.name, a) for a in k.resources.asn) + + def dump_children(self, fn): + """ + Write children CSV file. + """ + self.csvout(fn).writerows((k.name, k.resources.valid_until, k.path("bpki/resources/ca.cer")) + for k in self.kids) + + def dump_parents(self, fn): + """ + Write parents CSV file. + """ + if self.is_root(): + self.csvout(fn).writerow(("rootd", + "https://localhost:%d/" % self.rootd_port, + self.path("bpki/servers/ca.cer"), + self.path("bpki/servers/ca.cer"), + self.name, + self.sia_base)) + else: + parent_host = self.parent.hosted_by if self.parent.is_hosted() else self.parent + self.csvout(fn).writerow((self.parent.name, + self.up_down_url(), + self.parent.path("bpki/resources/ca.cer"), + parent_host.path("bpki/servers/ca.cer"), + self.name, + self.sia_base)) + + def dump_prefixes(self, fn): + """ + Write prefixes CSV file. + """ + f = self.csvout(fn) + for k in self.kids: + f.writerows((k.name, p) for p in (k.resources.v4 + k.resources.v6)) + + def dump_roas(self, fn): + """ + Write ROA CSV file. + """ + group = self.name if self.is_root() else self.parent.name + f = self.csvout(fn) + for r in self.roa_requests: + f.writerows((p, r.asn, group) + for p in (r.v4 + r.v6 if r.v4 and r.v6 else r.v4 or r.v6 or ())) + + def dump_clients(self, fn, db): + """ + Write pubclients CSV file. + """ + if self.runs_pubd(): + f = self.csvout(fn) + f.writerows((s.client_handle, s.path("bpki/resources/ca.cer"), s.sia_base) + for s in (db if only_one_pubd else [self] + self.kids)) + + def find_pubd(self, want_path = False): + """ + Walk up tree until we find somebody who runs pubd. + """ + s = self + path = [s] + while not s.runs_pubd(): + s = s.parent + path.append(s) + if want_path: + return s, ".".join(i.name for i in reversed(path)) + else: + return s + + def dump_conf(self, fn): + """ + Write configuration file for OpenSSL and RPKI tools. + """ + + s = self.find_pubd() + + r = { "handle" : self.name, + "run_pubd" : str(self.runs_pubd()), + "run_rootd" : str(self.is_root()), + "openssl" : prog_openssl, + "irdbd_sql_database" : "irdb%d" % self.engine, + "rpkid_sql_database" : "rpki%d" % self.engine, + "rpkid_server_host" : "localhost", + "rpkid_server_port" : str(self.rpkid_port), + "irdbd_server_host" : "localhost", + "irdbd_server_port" : str(self.irdbd_port), + "rootd_server_port" : str(self.rootd_port), + "pubd_sql_database" : "pubd%d" % self.engine, + "pubd_server_host" : "localhost", + "pubd_server_port" : str(s.pubd_port), + "publication_rsync_server" : "localhost:%s" % s.rsync_port } + + r.update(config_overrides) + + f = open(self.path(fn), "w") + f.write("# Automatically generated, do not edit\n") + print "Writing", f.name + + section = None + for line in open("examples/myrpki.conf"): + m = section_regexp.match(line) + if m: + section = m.group(1) + m = variable_regexp.match(line) + option = m.group(1) if m and section == "myrpki" else None + if option and option in r: + line = "%s = %s\n" % (option, r[option]) + f.write(line) + + f.close() + + def dump_rsyncd(self, fn): + """ + Write rsyncd configuration file. + """ + + if self.runs_pubd(): + f = open(self.path(fn), "w") + print "Writing", f.name + f.writelines(s + "\n" for s in + ("# Automatically generated, do not edit", + "port = %d" % self.rsync_port, + "address = localhost", + "[rpki]", + "log file = rsyncd.log", + "read only = yes", + "use chroot = no", + "path = %s" % self.path("publication"), + "comment = RPKI test")) + f.close() + + def run_myirbe(self): + """ + Run myirbe.py if this entity is not hosted by another engine. + """ + if not self.is_hosted(): + self.run_setup("configure_daemons", *[h.path("myrpki.xml") for h in self.hosts]) + + def run_myrpki(self): + """ + Run myrpki.py for this entity. + """ + self.run_setup("configure_resources") + + def run_setup(self, *args): + """ + Run setup.py for this entity. + """ + print 'Running "%s" for %s' % (" ".join(("myrpki",) + args), self.name) + subprocess.check_call(("python", prog_setup) + args, cwd = self.path()) + + def run_python_daemon(self, prog): + """ + Start a Python daemon and return a subprocess.Popen object + representing the running daemon. + """ + basename = os.path.basename(prog) + p = subprocess.Popen(("python", prog, "-d", "-c", self.path("myrpki.conf")), + cwd = self.path(), + stdout = open(self.path(os.path.splitext(basename)[0] + ".log"), "w"), + stderr = subprocess.STDOUT) + print "Running %s for %s: pid %d process %r" % (basename, self.name, p.pid, p) + return p + + def run_rpkid(self): + """ + Run rpkid. + """ + return self.run_python_daemon(prog_rpkid) + + def run_irdbd(self): + """ + Run irdbd. + """ + return self.run_python_daemon(prog_irdbd) + + def run_pubd(self): + """ + Run pubd. + """ + return self.run_python_daemon(prog_pubd) + + def run_rootd(self): + """ + Run rootd. + """ + return self.run_python_daemon(prog_rootd) + + def run_rsyncd(self): + """ + Run rsyncd. + """ + p = subprocess.Popen(("rsync", "--daemon", "--no-detach", "--config", "rsyncd.conf"), + cwd = self.path()) + print "Running rsyncd for %s: pid %d process %r" % (self.name, p.pid, p) + return p + + def run_openssl(self, *args, **kwargs): + """ + Run OpenSSL + """ + env = { "PATH" : os.environ["PATH"], + "BPKI_DIRECTORY" : self.path("bpki/servers"), + "OPENSSL_CONF" : "/dev/null", + "RANDFILE" : ".OpenSSL.whines.unless.I.set.this" } + env.update(kwargs) + subprocess.check_call((prog_openssl,) + args, cwd = self.path(), env = env) + + +os.environ["TZ"] = "UTC" +time.tzset() + +cfg_file = "yamltest.conf" +pidfile = None + +opts, argv = getopt.getopt(sys.argv[1:], "c:hp:?", ["config=", "help", "pidfile="]) +for o, a in opts: + if o in ("-h", "--help", "-?"): + print __doc__ + sys.exit(0) + if o in ("-c", "--config"): + cfg_file = a + elif o in ("-p", "--pidfile"): + pidfile = a + +# We can't usefully process more than one YAMl file at a time, so +# whine if there's more than one argument left. + +if len(argv) > 1: + raise RuntimeError, "Unexpected arguments %r" % argv + +try: + + if pidfile is not None: + open(pidfile, "w").write("%s\n" % os.getpid()) + + rpki.log.use_syslog = False + rpki.log.init("yamltest") + + yaml_file = argv[0] if argv else "../rpkid/tests/testbed.1.yaml" + + # Allow optional config file for this tool to override default + # passwords: this is mostly so that I can show a complete working + # example without publishing my own server's passwords. + + cfg = rpki.config.parser(cfg_file, "yamltest", allow_missing = True) + + only_one_pubd = cfg.getboolean("only_one_pubd", True) + prog_openssl = cfg.get("openssl", prog_openssl) + + config_overrides = dict( + (k, cfg.get(k)) + for k in ("rpkid_sql_password", "irdbd_sql_password", "pubd_sql_password", + "rpkid_sql_username", "irdbd_sql_username", "pubd_sql_username") + if cfg.has_option(k)) + + # Start clean + + for root, dirs, files in os.walk(test_dir, topdown = False): + for file in files: + os.unlink(os.path.join(root, file)) + for dir in dirs: + os.rmdir(os.path.join(root, dir)) + + # Read first YAML doc in file and process as compact description of + # test layout and resource allocations. Ignore subsequent YAML docs, + # they're for testbed.py, not this script. + + db = allocation_db(yaml.safe_load_all(open(yaml_file)).next()) + + # Show what we loaded + + db.dump() + + # Set up each entity in our test + + for d in db: + os.makedirs(d.path()) + d.dump_asns("asns.csv") + d.dump_prefixes("prefixes.csv") + d.dump_roas("roas.csv") + d.dump_conf("myrpki.conf") + d.dump_rsyncd("rsyncd.conf") + if False: + d.dump_children("children.csv") + d.dump_parents("parents.csv") + d.dump_clients("pubclients.csv", db) + + # Initialize BPKI and generate self-descriptor for each entity. + + for d in db: + d.run_setup("initialize") + + # This is where we need to get clever about running setup.py in its + # various modes to do the service URL and BPKI cross-certification + # setup. + + for d in db: + if d.is_root(): + print + d.run_setup("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.name)) + print + d.run_setup("configure_repository", d.path("entitydb", "pubclients", "%s.xml" % d.name)) + print + else: + print + d.parent.run_setup("configure_child", d.path("entitydb", "identity.xml")) + print + d.run_setup("configure_parent", d.parent.path("entitydb", "children", "%s.xml" % d.name)) + print + p, n = d.find_pubd(want_path = True) + p.run_setup("configure_publication_client", d.path("entitydb", "repositories", "%s.xml" % d.parent.name)) + print + d.run_setup("configure_repository", p.path("entitydb", "pubclients", "%s.xml" % n)) + print + + # Run myrpki.py several times for each entity. First pass misses + # stuff that isn't generated until later in first pass. Second pass + # should pick up everything and reach a stable state. If anything + # changes during third pass, that's a bug. + + for i in xrange(3): + for d in db: + d.run_myrpki() + + # Create publication directories. + + for d in db: + if d.is_root() or d.runs_pubd(): + os.makedirs(d.path("publication")) + + # Create RPKI root certificate. + + print "Creating rootd RPKI root certificate" + + # Should use req -subj here to set subject name. Later. + db.root.run_openssl("x509", "-req", "-sha256", "-outform", "DER", + "-signkey", "bpki/servers/ca.key", + "-in", "bpki/servers/ca.req", + "-out", "publication/root.cer", + "-extfile", "myrpki.conf", + "-extensions", "rootd_x509_extensions") + + # At this point we need to start a whole lotta daemons. + + progs = [] + + try: + print "Running daemons" + progs.append(db.root.run_rootd()) + progs.extend(d.run_irdbd() for d in db if not d.is_hosted()) + progs.extend(d.run_pubd() for d in db if d.runs_pubd()) + progs.extend(d.run_rsyncd() for d in db if d.runs_pubd()) + progs.extend(d.run_rpkid() for d in db if not d.is_hosted()) + + print "Giving daemons time to start up" + time.sleep(20) + + assert all(p.poll() is None for p in progs) + + # Run myirbe again for each host, to set up IRDB and RPKI objects. + # Need to run a second time to push BSC certs out to rpkid. Nothing + # should happen on the third pass. Oops, when hosting we need to + # run myrpki between myirbe passes, since only the hosted entity can + # issue the BSC, etc. + + for i in xrange(3): + for d in db: + d.run_myrpki() + for d in db: + d.run_myirbe() + + print "Done initializing daemons" + + # Wait until something terminates. + + signal.signal(signal.SIGCHLD, lambda *dont_care: None) + if all(p.poll() is None for p in progs): + signal.pause() + + finally: + + # Shut everything down. + + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + for p in progs: + if p.poll() is None: + os.kill(p.pid, signal.SIGTERM) + print "Program pid %d %r returned %d" % (p.pid, p, p.wait()) + +finally: + if pidfile is not None: + os.unlink(pidfile) |